public SnapServer CreateServer() { var settings = new HistorianServerDatabaseConfig("PPA", @"C:\Temp\Synchrophasor", true); var server = new SnapServer(settings); return(server); }
/// <summary> /// Releases the unmanaged resources used by the <see cref="SnapSocketListener"/> object and optionally releases the managed resources. /// </summary> /// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param> protected override void Dispose(bool disposing) { if (!m_disposed) { try { // This will be done regardless of whether the object is finalized or disposed. if (disposing) { m_isRunning = false; if (m_listener != null) { m_listener.Stop(); } m_server = null; lock (m_clients) { foreach (SnapNetworkServer client in m_clients) { client.Dispose(); } m_clients.Clear(); } // This will be done only when the object is disposed by calling Dispose(). } } finally { m_disposed = true; // Prevent duplicate dispose. base.Dispose(disposing); // Call base class Dispose(). } } }
/// <summary> /// Read historian data from server. /// </summary> /// <param name="instanceName">Historian instance name.</param> /// <param name="startTime">Start time of query.</param> /// <param name="stopTime">Stop time of query.</param> /// <param name="measurementIDs">Measurement IDs to query - or <c>null</c> for all available points.</param> /// <param name="resolution">Resolution for data query.</param> /// <param name="seriesLimit">Maximum number of points per series.</param> /// <param name="forceLimit">Flag that determines if series limit should be strictly enforced.</param> /// <param name="timestampType">Type of timestamps.</param> /// <returns>Enumeration of <see cref="TrendValue"/> instances read for time range.</returns> public IEnumerable <TrendValue> GetHistorianData(string instanceName, DateTime startTime, DateTime stopTime, ulong[] measurementIDs, Resolution resolution, int seriesLimit, bool forceLimit, TimestampType timestampType = TimestampType.UnixMilliseconds) { // Cancel any running operation CancellationToken cancellationToken = new CancellationToken(); Interlocked.Exchange(ref m_readCancellationToken, cancellationToken)?.Cancel(); SnapServer server = GetServer(instanceName)?.Host; IEnumerable <TrendValue> values = TrendValueAPI.GetHistorianData(server, instanceName, startTime, stopTime, measurementIDs, resolution, seriesLimit, forceLimit, cancellationToken); switch (timestampType) { case TimestampType.Ticks: return(values.Select(value => { value.Timestamp = value.Timestamp * 10000.0D + 621355968000000000.0D; return value; })); case TimestampType.UnixSeconds: return(values.Select(value => { value.Timestamp = value.Timestamp / 1000.0D; return value; })); default: return(values); } }
public void TestArchiveWriter() { Random r = new Random(3); _ = new EncodingDefinition(EncodingDefinition.FixedSizeIndividualGuid, EncodingDefinition.FixedSizeIndividualGuid); AdvancedServerDatabaseConfig <AmiKey, AmiKey> config = new AdvancedServerDatabaseConfig <AmiKey, AmiKey>("KV2CPQ", "C:\\Temp\\AMI", true); using (SnapServer server = new SnapServer(config)) { using (SnapClient client = SnapClient.Connect(server)) using (ClientDatabaseBase <AmiKey, AmiKey> db = client.GetDatabase <AmiKey, AmiKey>("KV2CPQ")) { int count = 10000000; Stopwatch sw = new Stopwatch(); sw.Start(); AmiKey key = new AmiKey(); AmiKey value = new AmiKey(); for (int x = count; x >= 0; x--) { key.Timestamp = (ulong)r.Next(); key.TableId = r.Next(); db.Write(key, value); } sw.Stop(); Console.WriteLine(count / sw.Elapsed.TotalSeconds / 1000000); Console.WriteLine(count); } } }
/// <summary> /// Creates a <see cref="SnapStreamingServer"/> /// </summary> protected void Initialize(SecureStreamServer <SocketUserPermissions> authentication, Stream stream, SnapServer server, bool requireSsl) { RequireSSL = requireSsl; m_rawStream = stream; m_authentication = authentication; m_server = server; }
protected override List <TimeSeriesValues> QueryTimeSeriesValues(DateTime startTime, DateTime stopTime, int maxDataPoints, Dictionary <ulong, string> targetMap, CancellationToken cancellationToken) { Dictionary <ulong, TimeSeriesValues> queriedTimeSeriesValues = new Dictionary <ulong, TimeSeriesValues>(); if (targetMap.Count > 0) { SnapServer server = GetAdapterInstance(InstanceName)?.Server?.Host; if ((object)server != null) { ulong[] measurementIDs = targetMap.Keys.ToArray(); Resolution resolution = TrendValueAPI.EstimatePlotResolution(InstanceName, startTime, stopTime, measurementIDs); using (SnapClient connection = SnapClient.Connect(server)) using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(InstanceName)) { foreach (TrendValue trendValue in TrendValueAPI.GetHistorianData(database, startTime, stopTime, measurementIDs, resolution, maxDataPoints, false, (CompatibleCancellationToken)cancellationToken)) { queriedTimeSeriesValues.GetOrAdd((ulong)trendValue.ID, id => new TimeSeriesValues { target = targetMap[id], datapoints = new List <double[]>() }) .datapoints.Add(new[] { trendValue.Value, trendValue.Timestamp }); } } } } return(queriedTimeSeriesValues.Values.ToList()); }
public SnapNetworkServer(SecureStreamServer<SocketUserPermissions> authentication, TcpClient client, SnapServer server, bool requireSsl = false) { m_client = client; m_rawStream = new NetworkStream(m_client.Client); m_client.Client.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.NoDelay, true); Initialize(authentication, m_rawStream, server, requireSsl); }
/// <summary> /// Creates a <see cref="SnapSocketListener"/> /// </summary> /// <param name="settings"></param> /// <param name="server"></param> /// <param name="parent"></param> public SnapSocketListener(SnapSocketListenerSettings settings, SnapServer server) : base(MessageClass.Framework) { if ((object)server == null) { throw new ArgumentNullException("server"); } if ((object)settings == null) { throw new ArgumentNullException("settings"); } m_server = server; m_settings = settings.CloneReadonly(); m_settings.Validate(); m_authenticator = new SecureStreamServer <SocketUserPermissions>(); if (settings.DefaultUserCanRead || settings.DefaultUserCanWrite || settings.DefaultUserIsAdmin) { m_authenticator.SetDefaultUser(true, new SocketUserPermissions() { CanRead = settings.DefaultUserCanRead, CanWrite = settings.DefaultUserCanWrite, IsAdmin = settings.DefaultUserIsAdmin }); } foreach (var user in settings.Users) { m_authenticator.AddUserIntegratedSecurity(user, new SocketUserPermissions() { CanRead = true, CanWrite = true, IsAdmin = true }); } // TODO: Shouldn't we use GSF.Communications async library here for scalability? If not, why not? // TODO: I think async communication classes could pass NetworkBinaryStream to a handler like ProcessClient... // TODO: Communications library may need a simple modification... Check with S. Wills for thoughts here... m_isRunning = true; m_listener = new TcpListener(m_settings.LocalEndPoint); //m_listener.Server.DualMode = true; m_listener.Start(); //var socket = m_listener.AcceptSocketAsync(); //socket.ContinueWith(ProcessDataRequests); //socket.Start(); Log.Publish(MessageLevel.Info, "Constructor Called", "Listening on " + m_settings.LocalEndPoint.ToString()); Thread th = new Thread(ProcessDataRequests); th.IsBackground = true; th.Start(); }
public void TestWriteSpeed() { //Logger.ReportToConsole(VerboseLevel.All ^ VerboseLevel.DebugLow); //Logger.SetLoggingPath("c:\\temp\\"); Globals.MemoryPool.SetMaximumBufferSize(4000 * 1024 * 1024L); //Thread th = new Thread(WriteSpeed); //th.IsBackground = true; //th.Start(); //Quit = false; foreach (string file in Directory.GetFiles("c:\\temp\\benchmark\\", "*.*", SearchOption.AllDirectories)) { File.Delete(file); } //PointCount = 0; HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true); using (SnapServer engine = new SnapServer(settings)) using (SnapClient client = SnapClient.Connect(engine)) using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB")) { Thread.Sleep(100); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); Stopwatch sw = new Stopwatch(); sw.Start(); for (int x = 0; x < PointsToArchive; x++) { key.PointID = (ulong)x; //PointCount = x; db.Write(key, value); } double totalTime = sw.Elapsed.TotalSeconds; Console.WriteLine("Completed write test in {0:#,##0.00} seconds at {1:#,##0.00} points per second", totalTime, PointsToArchive / totalTime); } //Quit = true; //th.Join(); //Console.WriteLine("Time (sec)\tPoints"); //foreach (var kvp in PointSamples) //{ // Console.WriteLine(kvp.Key.ToString() + "\t" + kvp.Value.ToString()); //} GC.Collect(); GC.WaitForPendingFinalizers(); Thread.Sleep(100); }
public void CreateHistorian() { Logger.Console.Verbose = VerboseLevel.All; using (SnapServer server = CreateServer()) { } GC.Collect(); GC.WaitForPendingFinalizers(); Thread.Sleep(1000); }
/// <summary> /// Creates a new <see cref="ReportHistorianReader"/>. /// </summary> /// <param name="server">Snapserver to connect to <see cref="SnapServer"/>.</param> /// <param name="instanceName">Name of the instance to connect to.</param> /// <param name="startTime">Starttime.</param> /// <param name="endTime">Endtime.</param> /// <param name="frameRate">SamplingRate of the signal.</param> /// <param name="pointIDs">PointIDs to be collected.</param> public ReportHistorianReader(SnapServer server, string instanceName, DateTime startTime, DateTime endTime, int frameRate, IEnumerable <ulong> pointIDs) { m_client = SnapClient.Connect(server); m_database = m_client.GetDatabase <HistorianKey, HistorianValue>(instanceName); m_key = new HistorianKey(); m_value = new HistorianValue(); SeekFilterBase <HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(DataPoint.RoundTimestamp(startTime, frameRate), DataPoint.RoundTimestamp(endTime, frameRate)); MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(pointIDs); m_stream = m_database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter); }
public void AttachFile() { Logger.Console.Verbose = VerboseLevel.All; using (SnapServer server = CreateServer()) { using (SnapClient client = SnapClient.Connect(server)) { client.GetDatabase("PPA").AttachFilesOrPaths(new string[] { @"C:\Temp\Synchrophasor\Dir\File2.d2" }); } } }
/// <summary> /// Creates a <see cref="SnapSocketListener"/> /// </summary> /// <param name="settings"></param> /// <param name="server"></param> /// <param name="parent"></param> public SnapSocketListener(SnapSocketListenerSettings settings, SnapServer server) : base(MessageClass.Framework) { if ((object)server == null) { throw new ArgumentNullException(nameof(server)); } if ((object)settings == null) { throw new ArgumentNullException(nameof(settings)); } m_server = server; m_settings = settings.CloneReadonly(); m_settings.Validate(); m_authenticator = new SecureStreamServer <SocketUserPermissions>(); if (settings.DefaultUserCanRead || settings.DefaultUserCanWrite || settings.DefaultUserIsAdmin) { m_authenticator.SetDefaultUser(true, new SocketUserPermissions { CanRead = settings.DefaultUserCanRead, CanWrite = settings.DefaultUserCanWrite, IsAdmin = settings.DefaultUserIsAdmin }); } foreach (string user in settings.Users) { m_authenticator.AddUserIntegratedSecurity(user, new SocketUserPermissions { CanRead = true, CanWrite = true, IsAdmin = true }); } m_isRunning = true; m_listener = new TcpListener(m_settings.LocalEndPoint); m_listener.Server.DualMode = m_settings.LocalEndPoint.AddressFamily == AddressFamily.InterNetworkV6; m_listener.Start(); Log.Publish(MessageLevel.Info, "Constructor Called", $"Listening on {m_settings.LocalEndPoint}"); new Thread(ProcessDataRequests) { IsBackground = true }.Start(); }
public void ReadData() { Logger.Console.Verbose = VerboseLevel.All; using (SnapServer server = CreateServer()) { using (SnapClient client = SnapClient.Connect(server)) using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("PPA")) using (TreeStream <HistorianKey, HistorianValue> stream = db.Read(null, null, null)) { Console.WriteLine(stream.Count()); } } }
/// <summary> /// Creates a <see cref="SnapSocketListener"/> /// </summary> /// <param name="settings"></param> /// <param name="server"></param> /// <param name="parent"></param> public SnapSocketListener(SnapSocketListenerSettings settings, SnapServer server) : base(MessageClass.Framework) { if ((object)server == null) throw new ArgumentNullException("server"); if ((object)settings == null) throw new ArgumentNullException("settings"); m_server = server; m_settings = settings.CloneReadonly(); m_settings.Validate(); m_authenticator = new SecureStreamServer<SocketUserPermissions>(); if (settings.DefaultUserCanRead || settings.DefaultUserCanWrite || settings.DefaultUserIsAdmin) { m_authenticator.SetDefaultUser(true, new SocketUserPermissions() { CanRead = settings.DefaultUserCanRead, CanWrite = settings.DefaultUserCanWrite, IsAdmin = settings.DefaultUserIsAdmin }); } foreach (var user in settings.Users) { m_authenticator.AddUserIntegratedSecurity(user, new SocketUserPermissions() { CanRead = true, CanWrite = true, IsAdmin = true }); } // TODO: Shouldn't we use GSF.Communications async library here for scalability? If not, why not? // TODO: I think async communication classes could pass NetworkBinaryStream to a handler like ProcessClient... // TODO: Communications library may need a simple modification... Check with S. Wills for thoughts here... m_isRunning = true; m_listener = new TcpListener(m_settings.LocalEndPoint); //m_listener.Server.DualMode = true; m_listener.Start(); //var socket = m_listener.AcceptSocketAsync(); //socket.ContinueWith(ProcessDataRequests); //socket.Start(); Log.Publish(MessageLevel.Info, "Constructor Called", "Listening on " + m_settings.LocalEndPoint.ToString()); Thread th = new Thread(ProcessDataRequests); th.IsBackground = true; th.Start(); }
public void WriteData() { Logger.Console.Verbose = VerboseLevel.All; using (SnapServer server = CreateServer()) { using (SnapClient client = SnapClient.Connect(server)) using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("PPA")) { HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); key.TimestampAsDate = DateTime.Now; key.PointID = LittleEndian.ToUInt64(Guid.NewGuid().ToByteArray(), 0); db.Write(key, value); } } }
public void GetAllFiles() { Logger.Console.Verbose = VerboseLevel.All; using (SnapServer server = CreateServer()) { using (SnapClient client = SnapClient.Connect(server)) using (ClientDatabaseBase db = client.GetDatabase("PPA")) { foreach (ArchiveDetails f in db.GetAllAttachedFiles()) { Console.WriteLine("{0}MB {1} TO {2}; ID:{3} Name: {4}", (f.FileSize / 1024d / 1024d).ToString("0.0"), f.FirstKey, f.LastKey, f.Id, f.FileName); } } } }
/// <summary> /// Creates a new <see cref="HistorianServer"/> instance. /// </summary> public HistorianServer(int?port) { var server = new ServerSettings(); if (port.HasValue) { var settings = new SnapSocketListenerSettings() { LocalTcpPort = port.Value }; settings.DefaultUserCanRead = true; settings.DefaultUserCanWrite = true; settings.DefaultUserIsAdmin = true; server.Listeners.Add(settings); } // Maintain a member level list of all established archive database engines m_host = new SnapServer(server); }
public void TestWriteSpeedRandom() { Logger.Console.Verbose = VerboseLevel.All; Random r = new Random(1); Thread th = new Thread(WriteSpeed); th.IsBackground = true; th.Start(); Quit = false; foreach (string file in Directory.GetFiles("c:\\temp\\benchmark\\")) { File.Delete(file); } PointCount = 0; HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true); using (SnapServer engine = new SnapServer(settings)) using (SnapClient client = SnapClient.Connect(engine)) using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB")) { Thread.Sleep(100); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < 10000000; x++) { key.Timestamp = (ulong)r.Next(); key.PointID = (ulong)x; PointCount = x; db.Write(key, value); } } Quit = true; th.Join(); Console.WriteLine("Time (sec)\tPoints"); foreach (KeyValuePair <double, int> kvp in PointSamples) { Console.WriteLine(kvp.Key.ToString() + "\t" + kvp.Value.ToString()); } }
public void TestRollover() { Logger.Console.Verbose = VerboseLevel.All; Globals.MemoryPool.SetMaximumBufferSize(4000 * 1024 * 1024L); foreach (string file in Directory.GetFiles("c:\\temp\\Test\\", "*.*", SearchOption.AllDirectories)) { File.Delete(file); } PointCount = 0; HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\Test\\Main\\", true); settings.FinalWritePaths.Add("c:\\temp\\Test\\Rollover\\"); ulong time = (ulong)DateTime.Now.Ticks; using (SnapServer engine = new SnapServer(settings)) using (SnapClient client = SnapClient.Connect(engine)) using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB")) { Thread.Sleep(100); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < 100000000; x++) { if (x % 100 == 0) { Thread.Sleep(10); } key.Timestamp = time; time += TimeSpan.TicksPerMinute; db.Write(key, value); } } GC.Collect(); GC.WaitForPendingFinalizers(); Thread.Sleep(100); }
public void DetatchFiles() { Logger.Console.Verbose = VerboseLevel.All; using (SnapServer server = CreateServer()) { using (SnapClient client = SnapClient.Connect(server)) using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("PPA")) { using (TreeStream <HistorianKey, HistorianValue> stream = db.Read(null, null, null)) { Console.WriteLine(stream.Count()); } db.DetatchFiles(db.GetAllAttachedFiles().Select(x => x.Id).ToList()); using (TreeStream <HistorianKey, HistorianValue> stream = db.Read(null, null, null)) { Console.WriteLine(stream.Count()); } } } }
/// <summary> /// Creates a new <see cref="HistorianServer"/> instance. /// </summary> public HistorianServer(int?port, string networkInterfaceIP = null) { ServerSettings server = new ServerSettings(); if (port.HasValue || !string.IsNullOrWhiteSpace(networkInterfaceIP)) { SnapSocketListenerSettings settings = new SnapSocketListenerSettings { LocalTcpPort = port ?? SnapSocketListenerSettings.DefaultNetworkPort, LocalIpAddress = networkInterfaceIP, DefaultUserCanRead = true, DefaultUserCanWrite = true, DefaultUserIsAdmin = true }; server.Listeners.Add(settings); } // Maintain a member level list of all established archive database engines Host = new SnapServer(server); }
/// <summary> /// Releases the unmanaged resources used by the <see cref="SnapSocketListener"/> object and optionally releases the managed resources. /// </summary> /// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param> protected override void Dispose(bool disposing) { if (!m_disposed) { try { if (!disposing) { return; } m_isRunning = false; if (m_listener != null) { m_listener.Stop(); } m_server = null; lock (m_clients) { foreach (SnapNetworkServer client in m_clients) { client.Dispose(); } m_clients.Clear(); } } finally { m_disposed = true; // Prevent duplicate dispose. base.Dispose(disposing); // Call base class Dispose(). } } }
/// <summary> /// Read historian data from server. /// </summary> /// <param name="server">The server to use for the query.</param> /// <param name="instanceName">Name of the archive to be queried.</param> /// <param name="startTime">Start time of query.</param> /// <param name="stopTime">Stop time of query.</param> /// <param name="measurementIDs">Measurement IDs to query - or <c>null</c> for all available points.</param> /// <param name="resolution">Resolution for data query.</param> /// <param name="seriesLimit">Maximum number of points per series.</param> /// <param name="forceLimit">Flag that determines if series limit should be strictly enforced.</param> /// <param name="cancellationToken">Cancellation token for query.</param> /// <returns>Enumeration of <see cref="TrendValue"/> instances read for time range.</returns> public static IEnumerable <TrendValue> GetHistorianData(SnapServer server, string instanceName, DateTime startTime, DateTime stopTime, ulong[] measurementIDs, Resolution resolution, int seriesLimit, bool forceLimit, ICancellationToken cancellationToken = null) { if (cancellationToken == null) { cancellationToken = new CancellationToken(); } if (server == null) { yield break; } // Setting series limit to zero requests full resolution data, which overrides provided parameter if (seriesLimit < 1) { resolution = Resolution.Full; } TimeSpan resolutionInterval = resolution.GetInterval(); SeekFilterBase <HistorianKey> timeFilter; MatchFilterBase <HistorianKey, HistorianValue> pointFilter = null; HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); // Set data scan resolution if (resolution == Resolution.Full) { timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime); } else { BaselineTimeInterval interval = BaselineTimeInterval.Second; if (resolutionInterval.Ticks < Ticks.PerMinute) { interval = BaselineTimeInterval.Second; } else if (resolutionInterval.Ticks < Ticks.PerHour) { interval = BaselineTimeInterval.Minute; } else if (resolutionInterval.Ticks == Ticks.PerHour) { interval = BaselineTimeInterval.Hour; } startTime = startTime.BaselinedTimestamp(interval); stopTime = stopTime.BaselinedTimestamp(interval); timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond)); } Dictionary <ulong, DataRow> metadata = null; using (SnapClient connection = SnapClient.Connect(server)) using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(instanceName)) { if (database == null) { yield break; } if (LocalOutputAdapter.Instances.TryGetValue(database.Info?.DatabaseName ?? DefaultInstanceName, out LocalOutputAdapter historianAdapter)) { metadata = historianAdapter?.Measurements; } if (metadata == null) { yield break; } // Setup point ID selections if (measurementIDs != null) { pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(measurementIDs); } else { measurementIDs = metadata.Keys.ToArray(); } // Start stream reader for the provided time window and selected points Dictionary <ulong, long> pointCounts = new Dictionary <ulong, long>(measurementIDs.Length); Dictionary <ulong, long> intervals = new Dictionary <ulong, long>(measurementIDs.Length); Dictionary <ulong, ulong> lastTimes = new Dictionary <ulong, ulong>(measurementIDs.Length); double range = (stopTime - startTime).TotalSeconds; ulong pointID, timestamp, resolutionSpan = (ulong)resolutionInterval.Ticks, baseTicks = (ulong)UnixTimeTag.BaseTicks.Value; long pointCount; if (resolutionSpan <= 1UL) { resolutionSpan = Ticks.PerSecond; } if (seriesLimit < 1) { seriesLimit = 1; } // Estimate total measurement counts per point so decimation intervals for each series can be calculated foreach (ulong measurementID in measurementIDs) { if (resolution == Resolution.Full) { pointCounts[measurementID] = metadata.TryGetValue(measurementID, out DataRow row) ? (long)(int.Parse(row["FramesPerSecond"].ToString()) * range) : 2; } else { pointCounts[measurementID] = (long)(range / resolutionInterval.TotalSeconds.NotZero(1.0D)); } } foreach (ulong measurementID in pointCounts.Keys) { intervals[measurementID] = (pointCounts[measurementID] / seriesLimit).NotZero(1L); } using (TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter)) { while (stream.Read(key, value) && !cancellationToken.IsCancelled) { pointID = key.PointID; timestamp = key.Timestamp; pointCount = pointCounts[pointID]; if (pointCount++ % intervals[pointID] == 0 || !forceLimit && timestamp - lastTimes.GetOrAdd(pointID, 0UL) > resolutionSpan) { yield return new TrendValue { ID = (long)pointID, Timestamp = (timestamp - baseTicks) / (double)Ticks.PerMillisecond, Value = value.AsSingle } } ; pointCounts[pointID] = pointCount; lastTimes[pointID] = timestamp; } } } } }
/// <summary> /// Starts a query that will read data source values, given a set of point IDs and targets, over a time range. /// </summary> /// <param name="startTime">Start-time for query.</param> /// <param name="stopTime">Stop-time for query.</param> /// <param name="interval">Interval from Grafana request.</param> /// <param name="includePeaks">Flag that determines if decimated data should include min/max interval peaks over provided time range.</param> /// <param name="targetMap">Set of IDs with associated targets to query.</param> /// <returns>Queried data source data in terms of value and time.</returns> protected override IEnumerable <DataSourceValue> QueryDataSourceValues(DateTime startTime, DateTime stopTime, string interval, bool includePeaks, Dictionary <ulong, string> targetMap) { SnapServer server = GetAdapterInstance(InstanceName)?.Server?.Host; if (server == null) { yield break; } using (SnapClient connection = SnapClient.Connect(server)) using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(InstanceName)) { if (database == null) { yield break; } if (!TryParseInterval(interval, out TimeSpan resolutionInterval)) { Resolution resolution = TrendValueAPI.EstimatePlotResolution(InstanceName, startTime, stopTime, targetMap.Keys); resolutionInterval = resolution.GetInterval(); } BaselineTimeInterval timeInterval = BaselineTimeInterval.Second; if (resolutionInterval.Ticks < Ticks.PerMinute) { timeInterval = BaselineTimeInterval.Second; } else if (resolutionInterval.Ticks < Ticks.PerHour) { timeInterval = BaselineTimeInterval.Minute; } else if (resolutionInterval.Ticks == Ticks.PerHour) { timeInterval = BaselineTimeInterval.Hour; } startTime = startTime.BaselinedTimestamp(timeInterval); stopTime = stopTime.BaselinedTimestamp(timeInterval); if (startTime == stopTime) { stopTime = stopTime.AddSeconds(1.0D); } SeekFilterBase <HistorianKey> timeFilter; // Set timestamp filter resolution if (includePeaks || resolutionInterval == TimeSpan.Zero) { // Full resolution query timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime); } else { // Interval query timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond)); } // Setup point ID selections MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(targetMap.Keys); Dictionary <ulong, ulong> lastTimes = new Dictionary <ulong, ulong>(targetMap.Count); Dictionary <ulong, Peak> peaks = new Dictionary <ulong, Peak>(targetMap.Count); ulong resolutionSpan = (ulong)resolutionInterval.Ticks; if (includePeaks) { resolutionSpan *= 2UL; } // Start stream reader for the provided time window and selected points using (TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter)) { HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); Peak peak = Peak.Default; while (stream.Read(key, value)) { ulong pointID = key.PointID; ulong timestamp = key.Timestamp; float pointValue = value.AsSingle; if (includePeaks) { peak = peaks.GetOrAdd(pointID, _ => new Peak()); peak.Set(pointValue, timestamp); } if (resolutionSpan > 0UL && timestamp - lastTimes.GetOrAdd(pointID, 0UL) < resolutionSpan) { continue; } // New value is ready for publication string target = targetMap[pointID]; MeasurementStateFlags flags = (MeasurementStateFlags)value.Value3; if (includePeaks) { if (peak.MinTimestamp > 0UL) { yield return(new DataSourceValue { Target = target, Value = peak.Min, Time = (peak.MinTimestamp - m_baseTicks) / (double)Ticks.PerMillisecond, Flags = flags }); } if (peak.MaxTimestamp != peak.MinTimestamp) { yield return(new DataSourceValue { Target = target, Value = peak.Max, Time = (peak.MaxTimestamp - m_baseTicks) / (double)Ticks.PerMillisecond, Flags = flags }); } peak.Reset(); } else { yield return(new DataSourceValue { Target = target, Value = pointValue, Time = (timestamp - m_baseTicks) / (double)Ticks.PerMillisecond, Flags = flags }); } lastTimes[pointID] = timestamp; } } } }
/// <summary> /// Starts a query that will read data source values, given a set of point IDs and targets, over a time range. /// </summary> /// <param name="startTime">Start-time for query.</param> /// <param name="stopTime">Stop-time for query.</param> /// <param name="interval">Interval from Grafana request.</param> /// <param name="decimate">Flag that determines if data should be decimated over provided time range.</param> /// <param name="targetMap">Set of IDs with associated targets to query.</param> /// <returns>Queried data source data in terms of value and time.</returns> protected override IEnumerable <DataSourceValue> QueryDataSourceValues(DateTime startTime, DateTime stopTime, string interval, bool decimate, Dictionary <ulong, string> targetMap) { SnapServer server = GetAdapterInstance(InstanceName)?.Server?.Host; if ((object)server == null) { yield break; } using (SnapClient connection = SnapClient.Connect(server)) using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(InstanceName)) { if ((object)database == null) { yield break; } Resolution resolution = TrendValueAPI.EstimatePlotResolution(InstanceName, startTime, stopTime, targetMap.Keys); SeekFilterBase <HistorianKey> timeFilter; // Set data scan resolution if (!decimate || resolution == Resolution.Full) { timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime); } else { TimeSpan resolutionInterval = resolution.GetInterval(); BaselineTimeInterval timeInterval = BaselineTimeInterval.Second; if (resolutionInterval.Ticks < Ticks.PerMinute) { timeInterval = BaselineTimeInterval.Second; } else if (resolutionInterval.Ticks < Ticks.PerHour) { timeInterval = BaselineTimeInterval.Minute; } else if (resolutionInterval.Ticks == Ticks.PerHour) { timeInterval = BaselineTimeInterval.Hour; } startTime = startTime.BaselinedTimestamp(timeInterval); stopTime = stopTime.BaselinedTimestamp(timeInterval); int milliseconds = 1; try { ConfigurationFile configFile = ConfigurationFile.Open(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile); CategorizedSettingsSection categorizedSettings = configFile.Settings; CategorizedSettingsElementCollection systemSettings = categorizedSettings["systemSettings"]; string val = systemSettings["HistoryTolerance"].Value; } catch { } // something went wrong, so just use original default timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond * milliseconds)); } // Setup point ID selections MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(targetMap.Keys); // Start stream reader for the provided time window and selected points using (TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter)) { HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); while (stream.Read(key, value)) { yield return(new DataSourceValue { Target = targetMap[key.PointID], Time = (key.Timestamp - m_baseTicks) / (double)Ticks.PerMillisecond, Value = value.AsSingle, Flags = (MeasurementStateFlags)value.Value3 }); } } } }
/// <summary> /// Releases the unmanaged resources used by the <see cref="SnapSocketListener"/> object and optionally releases the managed resources. /// </summary> /// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param> protected override void Dispose(bool disposing) { if (!m_disposed) { try { // This will be done regardless of whether the object is finalized or disposed. if (disposing) { m_isRunning = false; if (m_listener != null) m_listener.Stop(); m_server = null; lock (m_clients) { foreach (SnapNetworkServer client in m_clients) client.Dispose(); m_clients.Clear(); } // This will be done only when the object is disposed by calling Dispose(). } } finally { m_disposed = true; // Prevent duplicate dispose. base.Dispose(disposing); // Call base class Dispose(). } } }
/// <summary> /// Begins a new historian write operation. /// </summary> /// <param name="instanceName">Historian instance name.</param> /// <param name="values">Enumeration of <see cref="TrendValue"/> instances to write.</param> /// <param name="totalValues">Total values to write, if known in advance.</param> /// <param name="timestampType">Type of timestamps.</param> /// <returns>New operational state handle.</returns> public uint BeginHistorianWrite(string instanceName, IEnumerable <TrendValue> values, long totalValues = 0, TimestampType timestampType = TimestampType.UnixMilliseconds) { HistorianWriteOperationState operationState = new HistorianWriteOperationState { Total = totalValues }; uint operationHandle = Random.UInt32; while (!m_historianWriteOperationStates.TryAdd(operationHandle, operationState)) { operationHandle = Random.UInt32; } new Thread(() => { operationState.StartTime = DateTime.UtcNow.Ticks; try { SnapServer server = GetServer(instanceName)?.Host; if (server == null) { throw new InvalidOperationException($"Server is null for instance [{instanceName}]."); } using (SnapClient connection = SnapClient.Connect(server)) using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(instanceName)) { if (database == null) { throw new InvalidOperationException($"Database is null for instance [{instanceName}]."); } HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); foreach (TrendValue trendValue in values) { key.PointID = (ulong)trendValue.ID; switch (timestampType) { case TimestampType.Ticks: key.Timestamp = (ulong)trendValue.Timestamp; break; case TimestampType.UnixSeconds: key.Timestamp = (ulong)trendValue.Timestamp * 10000000UL + 621355968000000000UL; break; case TimestampType.UnixMilliseconds: key.Timestamp = (ulong)trendValue.Timestamp * 10000UL + 621355968000000000UL; break; } value.AsSingle = (float)trendValue.Value; database.Write(key, value); operationState.Progress++; if (operationState.CancellationToken.IsCancelled) { break; } } operationState.Completed = !operationState.CancellationToken.IsCancelled; } } catch (Exception ex) { operationState.Failed = true; operationState.FailedReason = ex.Message; } // Schedule operation handle to be removed CancelHistorianWrite(operationHandle); operationState.StopTime = DateTime.UtcNow.Ticks; }) { IsBackground = true } .Start(); return(operationHandle); }
public SnapNetworkServer(SecureStreamServer <SocketUserPermissions> authentication, TcpClient client, SnapServer server, bool requireSsl = false) { m_client = client; m_rawStream = new NetworkStream(m_client.Client); m_client.Client.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.NoDelay, true); Initialize(authentication, m_rawStream, server, requireSsl); }
/// <summary> /// Reads Data From the Historian and returns condensed DataPoints. /// </summary> /// <param name="start">Starttime.</param> /// <param name="end">EndTime.</param> /// <param name="measurements">Measurements to be read.</param> /// <param name="threshold">Threshhold used to determine number of alerts.</param> /// <param name="cancelationToken">Cancleation Token for the operation.</param> /// <param name="progress"> Progress Tracker <see cref="IProgress{T}"/>.</param> public IEnumerable <CondensedDataPoint> ReadCondensed(DateTime start, DateTime end, IEnumerable <ActiveMeasurement> measurements, double threshold, CancellationToken cancelationToken, IProgress <ulong> progress) { // Enumerate measurements once ActiveMeasurement[] activeMeasurements = measurements.ToArray(); List <CondensedDataPoint> result = new List <CondensedDataPoint>(activeMeasurements.Length); SnapServer server = GetServer(m_instance)?.Host; // start by separating all framerates foreach (int frameRate in activeMeasurements.Select(item => item.FramesPerSecond.GetValueOrDefault()).Distinct()) { ulong[] pointIDs = activeMeasurements.Where(item => item.FramesPerSecond == frameRate).Select(item => item.PointID).ToArray(); Dictionary <ulong, CondensedDataPoint> frameRateResult = new Dictionary <ulong, CondensedDataPoint>(); foreach (ulong key in pointIDs) { frameRateResult.Add(key, CondensedDataPoint.EmptyPoint(key)); if (cancelationToken.IsCancellationRequested) { return(result); } } using (ReportHistorianReader reader = new ReportHistorianReader(server, m_instance, start, end, frameRate, pointIDs)) { DataPoint point = new DataPoint(); // this.mProgress.Report(this.m_prevProgress); // Scan to first record if (!reader.ReadNext(point)) { throw new InvalidOperationException("No data for specified time range in openHistorian connection!"); } ulong currentTimeStamp = point.Timestamp; while (reader.ReadNext(point)) { if (currentTimeStamp < point.Timestamp) { progress.Report(point.Timestamp); } if (cancelationToken.IsCancellationRequested) { return(result); } if (!float.IsNaN(point.ValueAsSingle)) { if (frameRateResult.TryGetValue(point.PointID, out CondensedDataPoint dataPoint) && dataPoint != null) { if (point.ValueAsSingle > dataPoint.Max) { dataPoint.Max = point.ValueAsSingle; } if (point.ValueAsSingle < dataPoint.Min) { dataPoint.Min = point.ValueAsSingle; } dataPoint.Sum += point.ValueAsSingle; dataPoint.SqrSum += point.ValueAsSingle * point.ValueAsSingle; dataPoint.TotalPoints++; if (point.ValueAsSingle > threshold) { dataPoint.Alert++; } } } currentTimeStamp = point.Timestamp; } } result.AddRange(frameRateResult.Where(item => item.Value.TotalPoints > 0).Select(item => { item.Value.PointID = item.Key; return(item.Value); })); if (cancelationToken.IsCancellationRequested) { return(result); } } return(result); }
/// <summary> /// Creates a new <see cref="HistorianServer"/> /// </summary> public HistorianServer() { Host = new SnapServer(); }
public void VerifyDB() { //Logger.ReportToConsole(VerboseLevel.All ^ VerboseLevel.DebugLow); //Logger.ConsoleSubscriber.AddIgnored(Logger.LookupType("GSF.SortedTreeStore")); Globals.MemoryPool.SetMaximumBufferSize(1000 * 1024 * 1024); Globals.MemoryPool.SetTargetUtilizationLevel(TargetUtilizationLevels.Low); HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true); using (SnapServer engine = new SnapServer(settings)) using (SnapClient client = SnapClient.Connect(engine)) using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB")) using (TreeStream <HistorianKey, HistorianValue> scan = db.Read(null, null, null)) { HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); Stopwatch sw = new Stopwatch(); sw.Start(); for (int x = 0; x < PointsToArchive; x++) { if (!scan.Read(key, value)) { throw new Exception("Missing points"); } if (key.PointID != (ulong)x) { throw new Exception("Corrupt"); } if (key.Timestamp != 0) { throw new Exception("Corrupt"); } if (key.EntryNumber != 0) { throw new Exception("Corrupt"); } if (value.Value1 != 0) { throw new Exception("Corrupt"); } if (value.Value1 != 0) { throw new Exception("Corrupt"); } if (value.Value1 != 0) { throw new Exception("Corrupt"); } } double totalTime = sw.Elapsed.TotalSeconds; Console.WriteLine("Completed read test in {0:#,##0.00} seconds at {1:#,##0.00} points per second", totalTime, PointsToArchive / totalTime); if (scan.Read(key, value)) { throw new Exception("too many points"); } } }
public SnapStreamingServer(SecureStreamServer <SocketUserPermissions> authentication, Stream stream, SnapServer server, bool requireSsl = false) : base(MessageClass.Framework) { Initialize(authentication, stream, server, requireSsl); }