public void TestArchiveWriter()
{
Random r = new Random(3);
_ = new EncodingDefinition(EncodingDefinition.FixedSizeIndividualGuid, EncodingDefinition.FixedSizeIndividualGuid);
AdvancedServerDatabaseConfig <AmiKey, AmiKey> config = new AdvancedServerDatabaseConfig <AmiKey, AmiKey>("KV2CPQ", "C:\\Temp\\AMI", true);
using (SnapServer server = new SnapServer(config))
{
using (SnapClient client = SnapClient.Connect(server))
using (ClientDatabaseBase <AmiKey, AmiKey> db = client.GetDatabase <AmiKey, AmiKey>("KV2CPQ"))
{
int count = 10000000;
Stopwatch sw = new Stopwatch();
sw.Start();
AmiKey key = new AmiKey();
AmiKey value = new AmiKey();
for (int x = count; x >= 0; x--)
{
key.Timestamp = (ulong)r.Next();
key.TableId = r.Next();
db.Write(key, value);
}
sw.Stop();
Console.WriteLine(count / sw.Elapsed.TotalSeconds / 1000000);
Console.WriteLine(count);
}
}
}
protected override List <TimeSeriesValues> QueryTimeSeriesValues(DateTime startTime, DateTime stopTime, int maxDataPoints, Dictionary <ulong, string> targetMap, CancellationToken cancellationToken)
{
Dictionary <ulong, TimeSeriesValues> queriedTimeSeriesValues = new Dictionary <ulong, TimeSeriesValues>();
if (targetMap.Count > 0)
{
SnapServer server = GetAdapterInstance(InstanceName)?.Server?.Host;
if ((object)server != null)
{
ulong[] measurementIDs = targetMap.Keys.ToArray();
Resolution resolution = TrendValueAPI.EstimatePlotResolution(InstanceName, startTime, stopTime, measurementIDs);
using (SnapClient connection = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(InstanceName))
{
foreach (TrendValue trendValue in TrendValueAPI.GetHistorianData(database, startTime, stopTime, measurementIDs, resolution, maxDataPoints, false, (CompatibleCancellationToken)cancellationToken))
{
queriedTimeSeriesValues.GetOrAdd((ulong)trendValue.ID, id => new TimeSeriesValues {
target = targetMap[id], datapoints = new List <double[]>()
})
.datapoints.Add(new[] { trendValue.Value, trendValue.Timestamp });
}
}
}
}
return(queriedTimeSeriesValues.Values.ToList());
}
/// <summary>
/// Releases the unmanaged resources used by the <see cref="SnapNetworkServer"/> object and optionally releases the managed resources.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (!m_disposed)
{
try
{
// This will be done regardless of whether the object is finalized or disposed.
if (disposing)
{
// This will be done only when the object is disposed by calling Dispose().
if (m_host != null)
{
m_host.Dispose();
}
m_host = null;
}
}
finally
{
m_disposed = true; // Prevent duplicate dispose.
base.Dispose(disposing); // Call base class Dispose().
}
}
}
public SnapDBClient(SnapDBEngine engine, string instanceName)
{
m_client = SnapClient.Connect(engine.ServerHost);
m_database = m_client.GetDatabase <HistorianKey, HistorianValue>(instanceName);
m_key = new HistorianKey();
m_value = new HistorianValue();
m_lastKey = new HistorianKey();
}
public void TestWriteSpeed()
{
//Logger.ReportToConsole(VerboseLevel.All ^ VerboseLevel.DebugLow);
//Logger.SetLoggingPath("c:\\temp\\");
Globals.MemoryPool.SetMaximumBufferSize(4000 * 1024 * 1024L);
//Thread th = new Thread(WriteSpeed);
//th.IsBackground = true;
//th.Start();
//Quit = false;
foreach (string file in Directory.GetFiles("c:\\temp\\benchmark\\", "*.*", SearchOption.AllDirectories))
{
File.Delete(file);
}
//PointCount = 0;
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
using (SnapServer engine = new SnapServer(settings))
using (SnapClient client = SnapClient.Connect(engine))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB"))
{
Thread.Sleep(100);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
Stopwatch sw = new Stopwatch();
sw.Start();
for (int x = 0; x < PointsToArchive; x++)
{
key.PointID = (ulong)x;
//PointCount = x;
db.Write(key, value);
}
double totalTime = sw.Elapsed.TotalSeconds;
Console.WriteLine("Completed write test in {0:#,##0.00} seconds at {1:#,##0.00} points per second", totalTime, PointsToArchive / totalTime);
}
//Quit = true;
//th.Join();
//Console.WriteLine("Time (sec)\tPoints");
//foreach (var kvp in PointSamples)
//{
// Console.WriteLine(kvp.Key.ToString() + "\t" + kvp.Value.ToString());
//}
GC.Collect();
GC.WaitForPendingFinalizers();
Thread.Sleep(100);
}
public void AttachFile()
{
Logger.Console.Verbose = VerboseLevel.All;
using (var server = CreateServer())
{
using (var client = SnapClient.Connect(server))
{
client.GetDatabase("PPA").AttachFilesOrPaths(new string[] { @"C:\Temp\Synchrophasor\Dir\File2.d2" });
}
}
}
/// <summary>
/// Creates a new <see cref="ReportHistorianReader"/>.
/// </summary>
/// <param name="server">Snapserver to connect to <see cref="SnapServer"/>.</param>
/// <param name="instanceName">Name of the instance to connect to.</param>
/// <param name="startTime">Starttime.</param>
/// <param name="endTime">Endtime.</param>
/// <param name="frameRate">SamplingRate of the signal.</param>
/// <param name="pointIDs">PointIDs to be collected.</param>
public ReportHistorianReader(SnapServer server, string instanceName, DateTime startTime, DateTime endTime, int frameRate, IEnumerable <ulong> pointIDs)
{
m_client = SnapClient.Connect(server);
m_database = m_client.GetDatabase <HistorianKey, HistorianValue>(instanceName);
m_key = new HistorianKey();
m_value = new HistorianValue();
SeekFilterBase <HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(DataPoint.RoundTimestamp(startTime, frameRate), DataPoint.RoundTimestamp(endTime, frameRate));
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(pointIDs);
m_stream = m_database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter);
}
public void Start(string[] paths)
{
m_database = "PPA";
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig(m_database, null, false);
settings.ImportPaths.AddRange(paths);
m_localServer = new HistorianServer(settings);
HistorianQuery query = new HistorianQuery(SnapClient.Connect(m_localServer.Host));
m_updateFramework.Start(query);
m_updateFramework.Mode = ExecutionMode.Manual;
m_updateFramework.Enabled = true;
}
public void ReadData()
{
Logger.Console.Verbose = VerboseLevel.All;
using (var server = CreateServer())
{
using (var client = SnapClient.Connect(server))
using (var db = client.GetDatabase <HistorianKey, HistorianValue>("PPA"))
using (var stream = db.Read(null, null, null))
{
Console.WriteLine(stream.Count());
}
}
}
public void WriteData()
{
Logger.Console.Verbose = VerboseLevel.All;
using (var server = CreateServer())
{
using (var client = SnapClient.Connect(server))
using (var db = client.GetDatabase <HistorianKey, HistorianValue>("PPA"))
{
var key = new HistorianKey();
var value = new HistorianValue();
key.TimestampAsDate = DateTime.Now;
key.PointID = LittleEndian.ToUInt64(Guid.NewGuid().ToByteArray(), 0);
db.Write(key, value);
}
}
}
public void TestReadData()
{
using (var server = new HistorianServer(new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", false), 1234))
{
using (var client = SnapClient.Connect(server.Host))
{
var database = client.GetDatabase <HistorianKey, HistorianValue>("DB");
var stream = database.Read(10, 800 - 1);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
while (stream.Read(key, value))
{
Console.WriteLine(key.Timestamp);
}
}
}
}
public void GetAllFiles()
{
Logger.Console.Verbose = VerboseLevel.All;
using (var server = CreateServer())
{
using (var client = SnapClient.Connect(server))
using (var db = client.GetDatabase("PPA"))
{
foreach (var f in db.GetAllAttachedFiles())
{
Console.WriteLine("{0}MB {1} TO {2}; ID:{3} Name: {4}",
(f.FileSize / 1024d / 1024d).ToString("0.0"),
f.FirstKey, f.LastKey, f.Id, f.FileName);
}
}
}
}
private void btnPlot_Click(object sender, EventArgs e)
{
List <ulong> keys = new List <ulong>(chkAllPoints.CheckedItems.OfType <ulong>());
plot.Clear();
plot.AddInteraction(new PlotSurface2D.Interactions.HorizontalDrag());
plot.AddInteraction(new PlotSurface2D.Interactions.VerticalDrag());
plot.AddInteraction(new PlotSurface2D.Interactions.AxisDrag(false));
if (keys.Count == 0)
{
return;
}
var client = SnapClient.Connect(m_archiveFile.Host);
var db = client.GetDatabase <HistorianKey, HistorianValue>("");
Dictionary <ulong, SignalDataBase> results = db.GetSignals(0, ulong.MaxValue, keys, TypeSingle.Instance);
foreach (ulong point in keys)
{
List <double> y = new List <double>();
List <double> x = new List <double>();
SignalDataBase data = results[point];
for (int i = 0; i < data.Count; i++)
{
ulong time;
double value;
data.GetData(i, out time, out value);
x.Add(time);
y.Add(value);
}
LinePlot lines = new LinePlot(y, x);
plot.Add(lines);
}
plot.Refresh();
db.Dispose();
client.Dispose();
}
public void TestWriteSpeedRandom()
{
Logger.Console.Verbose = VerboseLevel.All;
Random r = new Random(1);
Thread th = new Thread(WriteSpeed);
th.IsBackground = true;
th.Start();
Quit = false;
foreach (string file in Directory.GetFiles("c:\\temp\\benchmark\\"))
{
File.Delete(file);
}
PointCount = 0;
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
using (SnapServer engine = new SnapServer(settings))
using (SnapClient client = SnapClient.Connect(engine))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB"))
{
Thread.Sleep(100);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < 10000000; x++)
{
key.Timestamp = (ulong)r.Next();
key.PointID = (ulong)x;
PointCount = x;
db.Write(key, value);
}
}
Quit = true;
th.Join();
Console.WriteLine("Time (sec)\tPoints");
foreach (KeyValuePair <double, int> kvp in PointSamples)
{
Console.WriteLine(kvp.Key.ToString() + "\t" + kvp.Value.ToString());
}
}
public void TestRollover()
{
Logger.Console.Verbose = VerboseLevel.All;
Globals.MemoryPool.SetMaximumBufferSize(4000 * 1024 * 1024L);
foreach (string file in Directory.GetFiles("c:\\temp\\Test\\", "*.*", SearchOption.AllDirectories))
{
File.Delete(file);
}
PointCount = 0;
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\Test\\Main\\", true);
settings.FinalWritePaths.Add("c:\\temp\\Test\\Rollover\\");
ulong time = (ulong)DateTime.Now.Ticks;
using (SnapServer engine = new SnapServer(settings))
using (SnapClient client = SnapClient.Connect(engine))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB"))
{
Thread.Sleep(100);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < 100000000; x++)
{
if (x % 100 == 0)
{
Thread.Sleep(10);
}
key.Timestamp = time;
time += TimeSpan.TicksPerMinute;
db.Write(key, value);
}
}
GC.Collect();
GC.WaitForPendingFinalizers();
Thread.Sleep(100);
}
/// <summary>
/// Releases the unmanaged resources used by the <see cref="SnapNetworkServer"/> object and optionally releases the managed resources.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (m_disposed)
{
return;
}
try
{
if (disposing)
{
m_host?.Dispose();
m_host = null;
}
}
finally
{
m_disposed = true; // Prevent duplicate dispose.
base.Dispose(disposing); // Call base class Dispose().
}
}
public void DetatchFiles()
{
Logger.Console.Verbose = VerboseLevel.All;
using (var server = CreateServer())
{
using (var client = SnapClient.Connect(server))
using (var db = client.GetDatabase <HistorianKey, HistorianValue>("PPA"))
{
using (var stream = db.Read(null, null, null))
{
Console.WriteLine(stream.Count());
}
db.DetatchFiles(db.GetAllAttachedFiles().Select(x => x.Id).ToList());
using (var stream = db.Read(null, null, null))
{
Console.WriteLine(stream.Count());
}
}
}
}
public void CreateAllDatabases()
{
Logger.Console.Verbose = VerboseLevel.All;
Array.ForEach(Directory.GetFiles(@"c:\temp\Scada\", "*.d2", SearchOption.AllDirectories), File.Delete);
Array.ForEach(Directory.GetFiles(@"c:\temp\Synchrophasor\", "*.d2", SearchOption.AllDirectories), File.Delete);
HistorianServerDatabaseConfig config1 = new HistorianServerDatabaseConfig("Scada", @"c:\temp\Scada\", true);
HistorianServerDatabaseConfig config2 = new HistorianServerDatabaseConfig("Synchrophasor", @"c:\temp\Synchrophasor\", true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (HistorianServer server = new HistorianServer())
{
server.AddDatabase(config1);
server.AddDatabase(config2);
using (SnapClient client = SnapClient.Connect(server.Host))
{
ClientDatabaseBase <HistorianKey, HistorianValue> database = client.GetDatabase <HistorianKey, HistorianValue>("Scada");
for (ulong x = 0; x < 10000; x++)
{
key.Timestamp = x;
database.Write(key, value);
}
database.HardCommit();
database = client.GetDatabase <HistorianKey, HistorianValue>("Synchrophasor");
for (ulong x = 0; x < 10000; x++)
{
key.Timestamp = x;
database.Write(key, value);
}
database.HardCommit();
}
}
}
public static bool TryGetSnapClient(string clientMac, out SnapClient snapclient)
{
var request = BuildSnapRequest("Client.GetStatus", new { id = clientMac });
string responseJson = SendSnapRequest(request, waitForResponse: true);
var response = JObject.Parse(responseJson);
if (response.ContainsKey("result") && ((JObject)response["result"]).ContainsKey("client"))
{
var client = (JObject)response["result"]["client"];
snapclient = new SnapClient()
{
Host = client["host"].Value <string>("name"),
Mac = client["host"].Value <string>("mac"),
Muted = client["config"]["volume"].Value <bool>("muted"),
Volume = client["config"]["volume"].Value <int>("percent")
};
return(true);
}
snapclient = null;
return(false);
}
private void BuildListOfAllPoints()
{
HashSet <ulong> keys = new HashSet <ulong>();
SnapClient client = SnapClient.Connect(m_archiveFile.Host);
ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("");
TreeStream <HistorianKey, HistorianValue> scanner = db.Read(0, ulong.MaxValue);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
while (scanner.Read(key, value))
{
keys.Add(key.PointID);
}
List <ulong> AllKeys = keys.ToList();
AllKeys.Sort();
chkAllPoints.Items.Clear();
AllKeys.ForEach((x) => chkAllPoints.Items.Add(x));
db.Dispose();
client.Dispose();
}
public void TestReadData()
{
var config1 = new HistorianServerDatabaseConfig("Scada", @"c:\temp\Scada\", true);
var config2 = new HistorianServerDatabaseConfig("Synchrophasor", @"c:\temp\Synchrophasor\", true);
using (HistorianServer server = new HistorianServer())
{
server.AddDatabase(config1);
server.AddDatabase(config2);
using (var client = SnapClient.Connect(server.Host))
{
var database = client.GetDatabase <HistorianKey, HistorianValue>("Scada");
TreeStream <HistorianKey, HistorianValue> stream = database.Read(0, 100);
stream.Dispose();
database = client.GetDatabase <HistorianKey, HistorianValue>("Synchrophasor");
stream = database.Read(0, 100);
stream.Dispose();
}
}
}
private SnapClient GetConnection(string instanceName)
{
SnapClient connection;
if (m_connections.TryGetValue(instanceName, out connection))
{
return(connection);
}
try
{
HistorianServer serverInstance = null;
LocalOutputAdapter historianAdapter;
if (LocalOutputAdapter.Instances.TryGetValue(instanceName, out historianAdapter))
{
serverInstance = historianAdapter?.Server;
}
if ((object)serverInstance == null)
{
return(null);
}
connection = SnapClient.Connect(serverInstance.Host);
}
catch (Exception ex)
{
LogException(new InvalidOperationException($"Failed to connect to historian \"{instanceName}\": {ex.Message}", ex));
}
if ((object)connection != null)
{
m_connections[instanceName] = connection;
}
return(connection);
}
public void TestRemoteAdapter()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("PPA", @"c:\temp\historian\", true);
using (HistorianServer server = new HistorianServer(settings))
using (SnapClient client = SnapClient.Connect(server.Host))
{
using (HistorianInputQueue queue = new HistorianInputQueue(() => client.GetDatabase <HistorianKey, HistorianValue>(string.Empty)))
{
for (uint x = 0; x < 100000; x++)
{
key.PointID = x;
queue.Enqueue(key, value);
}
Thread.Sleep(100);
}
Thread.Sleep(100);
}
//Thread.Sleep(100);
}
public void CreateScadaDatabase()
{
Logger.Console.Verbose = VerboseLevel.All;
Array.ForEach(Directory.GetFiles(@"c:\temp\Scada\", "*.d2", SearchOption.AllDirectories), File.Delete);
var key = new HistorianKey();
var value = new HistorianValue();
var settings = new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", true);
using (var server = new HistorianServer(settings))
using (var client = SnapClient.Connect(server.Host))
{
var database = client.GetDatabase <HistorianKey, HistorianValue>("db");
for (ulong x = 0; x < 1000; x++)
{
key.Timestamp = x;
database.Write(key, value);
}
database.HardCommit();
}
}
/// <summary>
/// Read historian data from server.
/// </summary>
/// <param name="server">The server to use for the query.</param>
/// <param name="instanceName">Name of the archive to be queried.</param>
/// <param name="startTime">Start time of query.</param>
/// <param name="stopTime">Stop time of query.</param>
/// <param name="measurementIDs">Measurement IDs to query - or <c>null</c> for all available points.</param>
/// <param name="resolution">Resolution for data query.</param>
/// <param name="seriesLimit">Maximum number of points per series.</param>
/// <param name="forceLimit">Flag that determines if series limit should be strictly enforced.</param>
/// <param name="cancellationToken">Cancellation token for query.</param>
/// <returns>Enumeration of <see cref="TrendValue"/> instances read for time range.</returns>
public static IEnumerable <TrendValue> GetHistorianData(SnapServer server, string instanceName, DateTime startTime, DateTime stopTime, ulong[] measurementIDs, Resolution resolution, int seriesLimit, bool forceLimit, ICancellationToken cancellationToken = null)
{
if (cancellationToken == null)
{
cancellationToken = new CancellationToken();
}
if (server == null)
{
yield break;
}
// Setting series limit to zero requests full resolution data, which overrides provided parameter
if (seriesLimit < 1)
{
resolution = Resolution.Full;
}
TimeSpan resolutionInterval = resolution.GetInterval();
SeekFilterBase <HistorianKey> timeFilter;
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = null;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
// Set data scan resolution
if (resolution == Resolution.Full)
{
timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
}
else
{
BaselineTimeInterval interval = BaselineTimeInterval.Second;
if (resolutionInterval.Ticks < Ticks.PerMinute)
{
interval = BaselineTimeInterval.Second;
}
else if (resolutionInterval.Ticks < Ticks.PerHour)
{
interval = BaselineTimeInterval.Minute;
}
else if (resolutionInterval.Ticks == Ticks.PerHour)
{
interval = BaselineTimeInterval.Hour;
}
startTime = startTime.BaselinedTimestamp(interval);
stopTime = stopTime.BaselinedTimestamp(interval);
timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond));
}
Dictionary <ulong, DataRow> metadata = null;
using (SnapClient connection = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(instanceName))
{
if (database == null)
{
yield break;
}
if (LocalOutputAdapter.Instances.TryGetValue(database.Info?.DatabaseName ?? DefaultInstanceName, out LocalOutputAdapter historianAdapter))
{
metadata = historianAdapter?.Measurements;
}
if (metadata == null)
{
yield break;
}
// Setup point ID selections
if (measurementIDs != null)
{
pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(measurementIDs);
}
else
{
measurementIDs = metadata.Keys.ToArray();
}
// Start stream reader for the provided time window and selected points
Dictionary <ulong, long> pointCounts = new Dictionary <ulong, long>(measurementIDs.Length);
Dictionary <ulong, long> intervals = new Dictionary <ulong, long>(measurementIDs.Length);
Dictionary <ulong, ulong> lastTimes = new Dictionary <ulong, ulong>(measurementIDs.Length);
double range = (stopTime - startTime).TotalSeconds;
ulong pointID, timestamp, resolutionSpan = (ulong)resolutionInterval.Ticks, baseTicks = (ulong)UnixTimeTag.BaseTicks.Value;
long pointCount;
if (resolutionSpan <= 1UL)
{
resolutionSpan = Ticks.PerSecond;
}
if (seriesLimit < 1)
{
seriesLimit = 1;
}
// Estimate total measurement counts per point so decimation intervals for each series can be calculated
foreach (ulong measurementID in measurementIDs)
{
if (resolution == Resolution.Full)
{
pointCounts[measurementID] = metadata.TryGetValue(measurementID, out DataRow row) ? (long)(int.Parse(row["FramesPerSecond"].ToString()) * range) : 2;
}
else
{
pointCounts[measurementID] = (long)(range / resolutionInterval.TotalSeconds.NotZero(1.0D));
}
}
foreach (ulong measurementID in pointCounts.Keys)
{
intervals[measurementID] = (pointCounts[measurementID] / seriesLimit).NotZero(1L);
}
using (TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter))
{
while (stream.Read(key, value) && !cancellationToken.IsCancelled)
{
pointID = key.PointID;
timestamp = key.Timestamp;
pointCount = pointCounts[pointID];
if (pointCount++ % intervals[pointID] == 0 || !forceLimit && timestamp - lastTimes.GetOrAdd(pointID, 0UL) > resolutionSpan)
{
yield return new TrendValue
{
ID = (long)pointID,
Timestamp = (timestamp - baseTicks) / (double)Ticks.PerMillisecond,
Value = value.AsSingle
}
}
;
pointCounts[pointID] = pointCount;
lastTimes[pointID] = timestamp;
}
}
}
}
}
public HistorianQuery(SnapClient historian)
{
m_historian = historian;
}
/// <summary>
/// Starts a query that will read data source values, given a set of point IDs and targets, over a time range.
/// </summary>
/// <param name="startTime">Start-time for query.</param>
/// <param name="stopTime">Stop-time for query.</param>
/// <param name="interval">Interval from Grafana request.</param>
/// <param name="includePeaks">Flag that determines if decimated data should include min/max interval peaks over provided time range.</param>
/// <param name="targetMap">Set of IDs with associated targets to query.</param>
/// <returns>Queried data source data in terms of value and time.</returns>
protected override IEnumerable <DataSourceValue> QueryDataSourceValues(DateTime startTime, DateTime stopTime, string interval, bool includePeaks, Dictionary <ulong, string> targetMap)
{
SnapServer server = GetAdapterInstance(InstanceName)?.Server?.Host;
if (server == null)
{
yield break;
}
using (SnapClient connection = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(InstanceName))
{
if (database == null)
{
yield break;
}
if (!TryParseInterval(interval, out TimeSpan resolutionInterval))
{
Resolution resolution = TrendValueAPI.EstimatePlotResolution(InstanceName, startTime, stopTime, targetMap.Keys);
resolutionInterval = resolution.GetInterval();
}
BaselineTimeInterval timeInterval = BaselineTimeInterval.Second;
if (resolutionInterval.Ticks < Ticks.PerMinute)
{
timeInterval = BaselineTimeInterval.Second;
}
else if (resolutionInterval.Ticks < Ticks.PerHour)
{
timeInterval = BaselineTimeInterval.Minute;
}
else if (resolutionInterval.Ticks == Ticks.PerHour)
{
timeInterval = BaselineTimeInterval.Hour;
}
startTime = startTime.BaselinedTimestamp(timeInterval);
stopTime = stopTime.BaselinedTimestamp(timeInterval);
if (startTime == stopTime)
{
stopTime = stopTime.AddSeconds(1.0D);
}
SeekFilterBase <HistorianKey> timeFilter;
// Set timestamp filter resolution
if (includePeaks || resolutionInterval == TimeSpan.Zero)
{
// Full resolution query
timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
}
else
{
// Interval query
timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond));
}
// Setup point ID selections
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(targetMap.Keys);
Dictionary <ulong, ulong> lastTimes = new Dictionary <ulong, ulong>(targetMap.Count);
Dictionary <ulong, Peak> peaks = new Dictionary <ulong, Peak>(targetMap.Count);
ulong resolutionSpan = (ulong)resolutionInterval.Ticks;
if (includePeaks)
{
resolutionSpan *= 2UL;
}
// Start stream reader for the provided time window and selected points
using (TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter))
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
Peak peak = Peak.Default;
while (stream.Read(key, value))
{
ulong pointID = key.PointID;
ulong timestamp = key.Timestamp;
float pointValue = value.AsSingle;
if (includePeaks)
{
peak = peaks.GetOrAdd(pointID, _ => new Peak());
peak.Set(pointValue, timestamp);
}
if (resolutionSpan > 0UL && timestamp - lastTimes.GetOrAdd(pointID, 0UL) < resolutionSpan)
{
continue;
}
// New value is ready for publication
string target = targetMap[pointID];
MeasurementStateFlags flags = (MeasurementStateFlags)value.Value3;
if (includePeaks)
{
if (peak.MinTimestamp > 0UL)
{
yield return(new DataSourceValue
{
Target = target,
Value = peak.Min,
Time = (peak.MinTimestamp - m_baseTicks) / (double)Ticks.PerMillisecond,
Flags = flags
});
}
if (peak.MaxTimestamp != peak.MinTimestamp)
{
yield return(new DataSourceValue
{
Target = target,
Value = peak.Max,
Time = (peak.MaxTimestamp - m_baseTicks) / (double)Ticks.PerMillisecond,
Flags = flags
});
}
peak.Reset();
}
else
{
yield return(new DataSourceValue
{
Target = target,
Value = pointValue,
Time = (timestamp - m_baseTicks) / (double)Ticks.PerMillisecond,
Flags = flags
});
}
lastTimes[pointID] = timestamp;
}
}
}
}
/// <summary>
/// Starts a query that will read data source values, given a set of point IDs and targets, over a time range.
/// </summary>
/// <param name="startTime">Start-time for query.</param>
/// <param name="stopTime">Stop-time for query.</param>
/// <param name="interval">Interval from Grafana request.</param>
/// <param name="decimate">Flag that determines if data should be decimated over provided time range.</param>
/// <param name="targetMap">Set of IDs with associated targets to query.</param>
/// <returns>Queried data source data in terms of value and time.</returns>
protected override IEnumerable <DataSourceValue> QueryDataSourceValues(DateTime startTime, DateTime stopTime, string interval, bool decimate, Dictionary <ulong, string> targetMap)
{
SnapServer server = GetAdapterInstance(InstanceName)?.Server?.Host;
if ((object)server == null)
{
yield break;
}
using (SnapClient connection = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(InstanceName))
{
if ((object)database == null)
{
yield break;
}
Resolution resolution = TrendValueAPI.EstimatePlotResolution(InstanceName, startTime, stopTime, targetMap.Keys);
SeekFilterBase <HistorianKey> timeFilter;
// Set data scan resolution
if (!decimate || resolution == Resolution.Full)
{
timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
}
else
{
TimeSpan resolutionInterval = resolution.GetInterval();
BaselineTimeInterval timeInterval = BaselineTimeInterval.Second;
if (resolutionInterval.Ticks < Ticks.PerMinute)
{
timeInterval = BaselineTimeInterval.Second;
}
else if (resolutionInterval.Ticks < Ticks.PerHour)
{
timeInterval = BaselineTimeInterval.Minute;
}
else if (resolutionInterval.Ticks == Ticks.PerHour)
{
timeInterval = BaselineTimeInterval.Hour;
}
startTime = startTime.BaselinedTimestamp(timeInterval);
stopTime = stopTime.BaselinedTimestamp(timeInterval);
int milliseconds = 1;
try
{
ConfigurationFile configFile = ConfigurationFile.Open(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile);
CategorizedSettingsSection categorizedSettings = configFile.Settings;
CategorizedSettingsElementCollection systemSettings = categorizedSettings["systemSettings"];
string val = systemSettings["HistoryTolerance"].Value;
}
catch { } // something went wrong, so just use original default
timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond * milliseconds));
}
// Setup point ID selections
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(targetMap.Keys);
// Start stream reader for the provided time window and selected points
using (TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter))
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
while (stream.Read(key, value))
{
yield return(new DataSourceValue
{
Target = targetMap[key.PointID],
Time = (key.Timestamp - m_baseTicks) / (double)Ticks.PerMillisecond,
Value = value.AsSingle,
Flags = (MeasurementStateFlags)value.Value3
});
}
}
}
}
public HistorianIArchive(HistorianServer server, string databaseName)
{
m_server = server;
m_client = SnapClient.Connect(m_server.Host);
m_clientDatabase = m_client.GetDatabase<HistorianKey, HistorianValue>(databaseName);
}
private async Task CopyModelAsCsvToStreamAsync(SecurityPrincipal securityPrincipal, NameValueCollection requestParameters, Stream responseStream, CancellationToken cancellationToken)
{
const double DefaultFrameRate = 30;
const int DefaultTimestampSnap = 0;
string dateTimeFormat = Program.Host.Model.Global.DateTimeFormat;
// TODO: Improve operation for large point lists:
// Pick-up "POST"ed parameters with a "genurl" param, then cache parameters
// in a memory cache and return the unique URL (a string instead of a file)
// with a "download" param and unique ID associated with cached parameters.
// Then extract params based on unique ID and follow normal steps...
// Note TSTolerance is in ms
string pointIDsParam = requestParameters["PointIDs"];
string startTimeParam = requestParameters["StartTime"];
string endTimeParam = requestParameters["EndTime"];
string timestampSnapParam = requestParameters["TSSnap"];
string frameRateParam = requestParameters["FrameRate"];
string alignTimestampsParam = requestParameters["AlignTimestamps"];
string missingAsNaNParam = requestParameters["MissingAsNaN"];
string fillMissingTimestampsParam = requestParameters["FillMissingTimestamps"];
string instanceName = requestParameters["InstanceName"];
string toleranceParam = requestParameters["TSTolerance"];
ulong[] pointIDs;
string headers;
if (string.IsNullOrEmpty(pointIDsParam))
{
throw new ArgumentNullException("PointIDs", "Cannot export data: no values were provided in \"PointIDs\" parameter.");
}
try
{
pointIDs = pointIDsParam.Split(',').Select(ulong.Parse).ToArray();
Array.Sort(pointIDs);
}
catch (Exception ex)
{
throw new ArgumentNullException("PointIDs", $"Cannot export data: failed to parse \"PointIDs\" parameter value \"{pointIDsParam}\": {ex.Message}");
}
if (string.IsNullOrEmpty(startTimeParam))
{
throw new ArgumentNullException("StartTime", "Cannot export data: no \"StartTime\" parameter value was specified.");
}
if (string.IsNullOrEmpty(pointIDsParam))
{
throw new ArgumentNullException("EndTime", "Cannot export data: no \"EndTime\" parameter value was specified.");
}
DateTime startTime, endTime;
try
{
startTime = DateTime.ParseExact(startTimeParam, dateTimeFormat, null, DateTimeStyles.AdjustToUniversal);
}
catch (Exception ex)
{
throw new ArgumentException($"Cannot export data: failed to parse \"StartTime\" parameter value \"{startTimeParam}\". Expected format is \"{dateTimeFormat}\". Error message: {ex.Message}", "StartTime", ex);
}
try
{
endTime = DateTime.ParseExact(endTimeParam, dateTimeFormat, null, DateTimeStyles.AdjustToUniversal);
}
catch (Exception ex)
{
throw new ArgumentException($"Cannot export data: failed to parse \"EndTime\" parameter value \"{endTimeParam}\". Expected format is \"{dateTimeFormat}\". Error message: {ex.Message}", "EndTime", ex);
}
if (startTime > endTime)
{
throw new ArgumentOutOfRangeException("StartTime", "Cannot export data: start time exceeds end time.");
}
using (DataContext dataContext = new DataContext())
{
// Validate current user has access to requested data
if (!dataContext.UserIsInRole(securityPrincipal, s_minimumRequiredRoles))
{
throw new SecurityException($"Cannot export data: access is denied for user \"{Thread.CurrentPrincipal.Identity?.Name ?? "Undefined"}\", minimum required roles = {s_minimumRequiredRoles.ToDelimitedString(", ")}.");
}
headers = GetHeaders(dataContext, pointIDs.Select(id => (int)id));
}
if (!double.TryParse(frameRateParam, out double frameRate))
{
frameRate = DefaultFrameRate;
}
if (!int.TryParse(timestampSnapParam, out int timestampSnap))
{
timestampSnap = DefaultTimestampSnap;
}
if (!double.TryParse(toleranceParam, out double tolerance))
{
tolerance = 0.5;
}
int toleranceTicks = (int)Math.Ceiling(tolerance * Ticks.PerMillisecond);
bool alignTimestamps = alignTimestampsParam?.ParseBoolean() ?? true;
bool missingAsNaN = missingAsNaNParam?.ParseBoolean() ?? true;
bool fillMissingTimestamps = alignTimestamps && (fillMissingTimestampsParam?.ParseBoolean() ?? false);
if (string.IsNullOrEmpty(instanceName))
{
instanceName = TrendValueAPI.DefaultInstanceName;
}
LocalOutputAdapter.Instances.TryGetValue(instanceName, out LocalOutputAdapter adapter);
HistorianServer serverInstance = adapter?.Server;
if (serverInstance == null)
{
throw new InvalidOperationException($"Cannot export data: failed to access internal historian server instance \"{instanceName}\".");
}
const int TargetBufferSize = 524288;
StringBuilder readBuffer = new StringBuilder(TargetBufferSize * 2);
ManualResetEventSlim bufferReady = new ManualResetEventSlim(false);
List <string> writeBuffer = new List <string>();
object writeBufferLock = new object();
bool readComplete = false;
Task readTask = Task.Factory.StartNew(() =>
{
try
{
using (SnapClient connection = SnapClient.Connect(serverInstance.Host))
{
Dictionary <ulong, int> pointIDIndex = new Dictionary <ulong, int>(pointIDs.Length);
float[] values = new float[pointIDs.Length];
for (int i = 0; i < pointIDs.Length; i++)
{
pointIDIndex.Add(pointIDs[i], i);
}
for (int i = 0; i < values.Length; i++)
{
values[i] = float.NaN;
}
ulong interval;
if (Math.Abs(frameRate % 1) <= (double.Epsilon * 100))
{
Ticks[] subseconds = Ticks.SubsecondDistribution((int)frameRate);
interval = (ulong)(subseconds.Length > 1 ? subseconds[1].Value : Ticks.PerSecond);
}
else
{
interval = (ulong)(Math.Floor(1.0d / frameRate) * Ticks.PerSecond);
}
ulong lastTimestamp = 0;
// Write data pages
SeekFilterBase <HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, endTime);
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(pointIDs);
HistorianKey historianKey = new HistorianKey();
HistorianValue historianValue = new HistorianValue();
// Write row values function
Action bufferValues = () =>
{
readBuffer.Append(missingAsNaN ? string.Join(",", values) : string.Join(",", values.Select(val => float.IsNaN(val) ? "" : $"{val}")));
if (readBuffer.Length < TargetBufferSize)
{
return;
}
lock (writeBufferLock)
writeBuffer.Add(readBuffer.ToString());
readBuffer.Clear();
bufferReady.Set();
};
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(instanceName))
{
// Start stream reader for the provided time window and selected points
TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter);
ulong timestamp = 0;
// Adjust timestamp to use first timestamp as base
bool adjustTimeStamp = true;
long baseTime = startTime.Ticks;
if (timestampSnap == 0)
{
adjustTimeStamp = false;
baseTime = Ticks.RoundToSecondDistribution(startTime.Ticks, frameRate, startTime.Ticks - startTime.Ticks % Ticks.PerSecond);
}
else if (timestampSnap == 1)
{
adjustTimeStamp = true;
}
else if (timestampSnap == 2)
{
adjustTimeStamp = false;
baseTime = startTime.Ticks;
}
while (stream.Read(historianKey, historianValue) && !cancellationToken.IsCancellationRequested)
{
if (alignTimestamps)
{
if (adjustTimeStamp)
{
adjustTimeStamp = false;
baseTime = (long)historianKey.Timestamp;
}
// Make sure the timestamp is actually close enough to the distribution
Ticks ticks = Ticks.ToSecondDistribution((long)historianKey.Timestamp, frameRate, baseTime, toleranceTicks);
if (ticks == Ticks.MinValue)
{
continue;
}
timestamp = (ulong)ticks.Value;
}
else
{
timestamp = historianKey.Timestamp;
}
// Start a new row for each encountered new timestamp
if (timestamp != lastTimestamp)
{
if (lastTimestamp > 0)
{
bufferValues();
}
for (int i = 0; i < values.Length; i++)
{
values[i] = float.NaN;
}
if (fillMissingTimestamps && lastTimestamp > 0 && timestamp > lastTimestamp)
{
ulong difference = timestamp - lastTimestamp;
if (difference > interval)
{
ulong interpolated = lastTimestamp;
for (ulong i = 1; i < difference / interval; i++)
{
interpolated = (ulong)Ticks.RoundToSecondDistribution((long)(interpolated + interval), frameRate, startTime.Ticks).Value;
readBuffer.Append($"{Environment.NewLine}{new DateTime((long)interpolated, DateTimeKind.Utc).ToString(dateTimeFormat)},");
bufferValues();
}
}
}
readBuffer.Append($"{Environment.NewLine}{new DateTime((long)timestamp, DateTimeKind.Utc).ToString(dateTimeFormat)},");
lastTimestamp = timestamp;
}
// Save value to its column
values[pointIDIndex[historianKey.PointID]] = historianValue.AsSingle;
}
if (timestamp > 0)
{
bufferValues();
}
if (readBuffer.Length > 0)
{
lock (writeBufferLock)
writeBuffer.Add(readBuffer.ToString());
}
}
}
}
finally
{
readComplete = true;
bufferReady.Set();
}
}, cancellationToken);
Task writeTask = Task.Factory.StartNew(() =>
{
using (StreamWriter writer = new StreamWriter(responseStream))
{
//Ticks exportStart = DateTime.UtcNow.Ticks;
string[] localBuffer;
// Write column headers
writer.Write(headers);
while ((writeBuffer.Count > 0 || !readComplete) && !cancellationToken.IsCancellationRequested)
{
bufferReady.Wait(cancellationToken);
bufferReady.Reset();
lock (writeBufferLock)
{
localBuffer = writeBuffer.ToArray();
writeBuffer.Clear();
}
foreach (string buffer in localBuffer)
{
writer.Write(buffer);
}
}
// Flush stream
writer.Flush();
//Debug.WriteLine("Export time: " + (DateTime.UtcNow.Ticks - exportStart).ToElapsedTimeString(3));
}
}, cancellationToken);
await readTask;
await writeTask;
}
public HistorianIArchive(HistorianServer server, string databaseName)
{
m_server = server;
m_client = SnapClient.Connect(m_server.Host);
m_clientDatabase = m_client.GetDatabase <HistorianKey, HistorianValue>(databaseName);
}
public HistorianQuery(SnapClient historian)
{
m_historian = historian;
}