/// <summary>
/// Initializes <see cref="SynchronizedClientSubscription"/>.
/// </summary>
public override void Initialize()
{
MeasurementKey[] inputMeasurementKeys;
string setting;
if (Settings.TryGetValue("inputMeasurementKeys", out setting))
{
// IMPORTANT: The allowSelect argument of ParseInputMeasurementKeys must be null
// in order to prevent SQL injection via the subscription filter expression
inputMeasurementKeys = AdapterBase.ParseInputMeasurementKeys(DataSource, false, setting);
m_requestedInputFilter = setting;
// IMPORTANT: We need to remove the setting before calling base.Initialize()
// or else we will still be subject to SQL injection
Settings.Remove("inputMeasurementKeys");
}
else
{
inputMeasurementKeys = new MeasurementKey[0];
m_requestedInputFilter = null;
}
base.Initialize();
// Set the InputMeasurementKeys and UsePrecisionTimer properties after calling
// base.Initialize() so that the base class does not overwrite our settings
InputMeasurementKeys = inputMeasurementKeys;
UsePrecisionTimer = false;
if (Settings.TryGetValue("bufferBlockRetransmissionTimeout", out setting))
{
m_bufferBlockRetransmissionTimeout = double.Parse(setting);
}
else
{
m_bufferBlockRetransmissionTimeout = 5.0D;
}
if (Settings.TryGetValue("requestNaNValueFilter", out setting))
{
m_isNaNFiltered = m_parent.AllowNaNValueFilter && setting.ParseBoolean();
}
else
{
m_isNaNFiltered = false;
}
m_bufferBlockRetransmissionTimer = Common.TimerScheduler.CreateTimer((int)(m_bufferBlockRetransmissionTimeout * 1000.0D));
m_bufferBlockRetransmissionTimer.AutoReset = false;
m_bufferBlockRetransmissionTimer.Elapsed += BufferBlockRetransmissionTimer_Elapsed;
// Handle temporal session initialization
if (this.TemporalConstraintIsDefined())
{
m_iaonSession = this.CreateTemporalSession();
}
}
public IMeasurement[] GetMeasurements(string filterExpression)
{
MeasurementKey[] keys = AdapterBase.ParseInputMeasurementKeys(m_dataSource, false, filterExpression);
IMeasurement measurement = null;
return(keys
.Where(key => m_measurementLookup.TryGetValue(key, out measurement))
.Select(key => measurement)
.ToArray());
}
public MeasurementKey GetMeasurementKey(string filterExpression)
{
MeasurementKey[] keys = AdapterBase.ParseInputMeasurementKeys(m_dataSource, false, filterExpression);
if (keys.Length > 1)
{
throw new InvalidOperationException($"Ambiguous filter returned {keys.Length} measurement keys: {filterExpression}.");
}
if (keys.Length == 0)
{
return(MeasurementKey.Undefined);
}
return(keys[0]);
}
public virtual void RefreshMetadata()
{
// Force a recalculation of input measurement keys so that system can appropriately update routing tables
string setting;
if (Settings.TryGetValue("inputMeasurementKeys", out setting))
{
InputMeasurementKeys = AdapterBase.ParseInputMeasurementKeys(DataSource, setting);
}
else
{
InputMeasurementKeys = null;
}
InputSourceIDs = InputSourceIDs;
}
public IMeasurement GetMeasurement(string filterExpression)
{
MeasurementKey[] keys = AdapterBase.ParseInputMeasurementKeys(m_dataSource, false, filterExpression);
IMeasurement measurement;
if (keys.Length > 1)
{
throw new InvalidOperationException($"Ambiguous filter returned {keys.Length} measurements: {filterExpression}.");
}
if (keys.Length == 0 || !m_measurementLookup.TryGetValue(keys[0], out measurement))
{
return(null);
}
return(measurement);
}
/// <summary>
/// Parses connection string. Derived classes should override for custom connection string parsing.
/// </summary>
/// <param name="instance">Target <see cref="IIndependentAdapterManager"/> instance.</param>
public static void HandleParseConnectionString(this IIndependentAdapterManager instance)
{
// Parse all properties marked with ConnectionStringParameterAttribute from provided ConnectionString value
ConnectionStringParser parser = new ConnectionStringParser();
parser.ParseConnectionString(instance.ConnectionString, instance);
// Parse input measurement keys like class was a typical adapter
if (instance.Settings.TryGetValue(nameof(instance.InputMeasurementKeys), out string setting))
{
instance.InputMeasurementKeys = AdapterBase.ParseInputMeasurementKeys(instance.DataSource, true, setting, instance.SourceMeasurementTable);
}
// Parse output measurement keys like class was a typical adapter
if (instance.Settings.TryGetValue(nameof(instance.OutputMeasurements), out setting))
{
instance.OutputMeasurements = AdapterBase.ParseOutputMeasurements(instance.DataSource, true, setting, instance.SourceMeasurementTable);
}
}
private void AssignInputMeasurements(Dictionary <string, string> settings)
{
if (!settings.ContainsKey("inputMeasurementKeys") && settings.TryGetValue("variableList", out string variableList))
{
Dictionary <string, string> variables = variableList.ParseKeyValuePairs();
if (variables.Count > 0)
{
settings["inputMeasurementKeys"] = string.Join(";", variables.Values);
}
}
MeasurementKey[] inputMeasurementKeys = settings.TryGetValue("inputMeasurementKeys", out string setting) ?
AdapterBase.ParseInputMeasurementKeys(m_dataSource, false, setting) :
Array.Empty <MeasurementKey>();
m_inputSignalIDs = inputMeasurementKeys.Select(k => k.SignalID).ToArray();
LoadMeasurements(m_inputSignalIDs, dataGridViewInputMeasurements, groupBoxInputMeasurements);
}
private List <MetadataRecord> GetMetadata()
{
Ticks operationTime;
Ticks operationStartTime;
// Load historian meta-data
ShowMessage(">>> Loading source connection metadata...");
operationStartTime = DateTime.UtcNow.Ticks;
List <MetadataRecord> metadata = MetadataRecord.Query(m_settings.HostAddress, m_settings.MetadataPort, m_settings.MetadataTimeout);
operationTime = DateTime.UtcNow.Ticks - operationStartTime;
ShowMessage("*** Metadata Load Complete ***");
ShowMessage($"Total metadata load time {operationTime.ToElapsedTimeString(3)}...");
// Parse meta-data expression
ShowMessage(">>> Processing filter expression for metadata...");
operationStartTime = DateTime.UtcNow.Ticks;
MeasurementKey[] inputKeys = AdapterBase.ParseInputMeasurementKeys(MetadataRecord.Metadata, false, m_settings.PointList, "MeasurementDetail");
List <ulong> pointIDList = inputKeys.Select(key => (ulong)key.ID).ToList();
List <MetadataRecord> records = new List <MetadataRecord>();
foreach (ulong pointID in pointIDList)
{
MetadataRecord record = metadata.FirstOrDefault(md => md.PointID == pointID);
if ((object)record != null)
{
records.Add(record);
}
}
operationTime = DateTime.UtcNow.Ticks - operationStartTime;
ShowMessage($">>> Historian read will be for {pointIDList.Count:N0} points based on provided meta-data expression.");
ShowMessage("*** Filter Expression Processing Complete ***");
ShowMessage($"Total filter expression processing time {operationTime.ToElapsedTimeString(3)}...");
return(records);
}
/// <summary>
/// Determines if subscriber has rights to specified <paramref name="signalID"/>.
/// </summary>
/// <param name="signalID"><see cref="Guid"/> signal ID to lookup.</param>
/// <returns><c>true</c> if subscriber has rights to specified <paramref name="signalID"/>; otherwise <c>false</c>.</returns>
public bool SubscriberHasRights(Guid signalID)
{
// TODO: Abstract publisher ACL logic -- except subscriber enabled flag does not need to be checked
const string FilterRegex = @"(ALLOW|DENY)\s+WHERE\s+([^;]*)";
DataRow subscriber;
DataRow[] subscriberMeasurementGroups;
IEnumerable <bool> explicitAuthorizationFlags;
IEnumerable <bool> explicitGroupAuthorizationFlags;
IEnumerable <bool> implicitFilterAuthorizationFlags;
bool explicitlyAuthorized = false;
bool explicitlyAuthorizedByGroup = false;
bool implicitlyAuthorizedByFilter = false;
// This is different from the ACL logic used by the system --
// because we are only calculating effective permissions to aid with configuration,
// we do not care whether the Subscriber is enabled
subscriber = m_subscriberPermissionsDataSet.Tables["Subscribers"].Select(string.Format("ID = '{0}'", CurrentItem.ID)).FirstOrDefault();
// If subscriber has been disabled or removed
// from the list of valid subscribers,
// they no longer have rights to any signals
if ((object)subscriber == null)
{
return(false);
}
// Look up explicitly defined individual measurements
explicitAuthorizationFlags = m_subscriberPermissionsDataSet.Tables["SubscriberMeasurements"].Select(string.Format("SubscriberID = '{0}' AND SignalID = '{1}'", CurrentItem.ID, signalID))
.Select(measurement => measurement["Allowed"].ToNonNullString("0").ParseBoolean());
foreach (bool flag in explicitAuthorizationFlags)
{
if (flag)
{
explicitlyAuthorized = true;
}
else
{
return(false);
}
}
if (explicitlyAuthorized)
{
return(true);
}
// Look up explicitly defined group based measurements
subscriberMeasurementGroups = m_subscriberPermissionsDataSet.Tables["SubscriberMeasurementGroups"].Select(string.Format("SubscriberID = '{0}'", CurrentItem.ID));
explicitGroupAuthorizationFlags = subscriberMeasurementGroups
.Where(subscriberMeasurementGroup => m_subscriberPermissionsDataSet.Tables["MeasurementGroupMeasurements"].Select(string.Format("SignalID = '{0}' AND MeasurementGroupID = {1}", signalID, subscriberMeasurementGroup["MeasurementGroupID"])).Length > 0)
.Select(subscriberMeasurementGroup => subscriberMeasurementGroup["Allowed"].ToNonNullString("0").ParseBoolean());
foreach (bool flag in explicitGroupAuthorizationFlags)
{
if (flag)
{
explicitlyAuthorizedByGroup = true;
}
else
{
return(false);
}
}
if (explicitlyAuthorizedByGroup)
{
return(true);
}
// Look up implicitly defined filter based measurements
implicitFilterAuthorizationFlags = Regex.Matches(subscriber["AccessControlFilter"].ToNonNullString().ReplaceControlCharacters(), FilterRegex, RegexOptions.IgnoreCase)
.Cast <Match>()
.Where(match => m_subscriberPermissionsDataSet.Tables["ActiveMeasurements"].Select(string.Format("SignalID = '{0}' AND ({1})", signalID, match.Groups[2].Value)).Length > 0)
.Select(match => (match.Groups[1].Value == "ALLOW"));
foreach (bool flag in implicitFilterAuthorizationFlags)
{
if (flag)
{
implicitlyAuthorizedByFilter = true;
}
else
{
return(false);
}
}
if (implicitlyAuthorizedByFilter)
{
return(true);
}
// Look up implicitly defined group based measurements
return(subscriberMeasurementGroups
.Select(subscriberMeasurementGroup => Tuple.Create(subscriberMeasurementGroup, m_subscriberPermissionsDataSet.Tables["MeasurementGroups"].Select(string.Format("ID = {0}", subscriberMeasurementGroup["MeasurementGroupID"]))))
.Where(tuple => tuple.Item2.Any(measurementGroup => AdapterBase.ParseInputMeasurementKeys(m_subscriberPermissionsDataSet, false, measurementGroup["FilterExpression"].ToNonNullString()).Select(key => key.SignalID).Contains(signalID)))
.Select(tuple => tuple.Item1["Allowed"].ToNonNullString("0").ParseBoolean())
.DefaultIfEmpty(false)
.All(allowed => allowed));
}
public MeasurementKey[] GetMeasurementKeys(string filterExpression)
{
return(AdapterBase.ParseInputMeasurementKeys(m_dataSource, false, filterExpression));
}
private IEnumerable <DataSourceValueGroup> QueryTarget(Target sourceTarget, string queryExpression, DateTime startTime, DateTime stopTime, string interval, bool decimate, bool dropEmptySeries, CancellationToken cancellationToken)
{
if (queryExpression.ToLowerInvariant().Contains(DropEmptySeriesCommand))
{
dropEmptySeries = true;
queryExpression = queryExpression.ReplaceCaseInsensitive(DropEmptySeriesCommand, "");
}
// A single target might look like the following:
// PPA:15; STAT:20; SETSUM(COUNT(PPA:8; PPA:9; PPA:10)); FILTER ActiveMeasurements WHERE SignalType IN ('IPHA', 'VPHA'); RANGE(PPA:99; SUM(FILTER ActiveMeasurements WHERE SignalType = 'FREQ'; STAT:12))
HashSet <string> targetSet = new HashSet <string>(new[] { queryExpression }, StringComparer.OrdinalIgnoreCase); // Targets include user provided input, so casing should be ignored
HashSet <string> reducedTargetSet = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
List <Match> seriesFunctions = new List <Match>();
foreach (string target in targetSet)
{
// Find any series functions in target
Match[] matchedFunctions = TargetCache <Match[]> .GetOrAdd(target, () =>
s_seriesFunctions.Matches(target).Cast <Match>().ToArray());
if (matchedFunctions.Length > 0)
{
seriesFunctions.AddRange(matchedFunctions);
// Reduce target to non-function expressions - important so later split on ';' succeeds properly
string reducedTarget = target;
foreach (string expression in matchedFunctions.Select(match => match.Value))
{
reducedTarget = reducedTarget.Replace(expression, "");
}
if (!string.IsNullOrWhiteSpace(reducedTarget))
{
reducedTargetSet.Add(reducedTarget);
}
}
else
{
reducedTargetSet.Add(target);
}
}
if (seriesFunctions.Count > 0)
{
// Execute series functions
foreach (Tuple <SeriesFunction, string, GroupOperation> parsedFunction in seriesFunctions.Select(ParseSeriesFunction))
{
foreach (DataSourceValueGroup valueGroup in ExecuteSeriesFunction(sourceTarget, parsedFunction, startTime, stopTime, interval, decimate, dropEmptySeries, cancellationToken))
{
yield return(valueGroup);
}
}
// Use reduced target set that excludes any series functions
targetSet = reducedTargetSet;
}
// Query any remaining targets
if (targetSet.Count > 0)
{
// Split remaining targets on semi-colon, this way even multiple filter expressions can be used as inputs to functions
string[] allTargets = targetSet.Select(target => target.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)).SelectMany(currentTargets => currentTargets).ToArray();
// Expand target set to include point tags for all parsed inputs
foreach (string target in allTargets)
{
targetSet.UnionWith(TargetCache <string[]> .GetOrAdd(target, () => AdapterBase.ParseInputMeasurementKeys(Metadata, false, target).Select(key => key.TagFromKey(Metadata)).ToArray()));
}
Dictionary <ulong, string> targetMap = new Dictionary <ulong, string>();
// Target set now contains both original expressions and newly parsed individual point tags - to create final point list we
// are only interested in the point tags, provided either by direct user entry or derived by parsing filter expressions
foreach (string target in targetSet)
{
// Reduce all targets down to a dictionary of point ID's mapped to point tags
MeasurementKey key = TargetCache <MeasurementKey> .GetOrAdd(target, () => target.KeyFromTag(Metadata));
if (key == MeasurementKey.Undefined)
{
Tuple <MeasurementKey, string> result = TargetCache <Tuple <MeasurementKey, string> > .GetOrAdd($"signalID@{target}", () => target.KeyAndTagFromSignalID(Metadata));
key = result.Item1;
string pointTag = result.Item2;
if (key == MeasurementKey.Undefined)
{
result = TargetCache <Tuple <MeasurementKey, string> > .GetOrAdd($"key@{target}", () =>
{
MeasurementKey.TryParse(target, out MeasurementKey parsedKey);
return(new Tuple <MeasurementKey, string>(parsedKey, parsedKey.TagFromKey(Metadata)));
});
key = result.Item1;
pointTag = result.Item2;
if (key != MeasurementKey.Undefined)
{
targetMap[key.ID] = pointTag;
}
}
else
{
targetMap[key.ID] = pointTag;
}
}
else
{
targetMap[key.ID] = target;
}
}
// Query underlying data source for each target - to prevent parallel read from data source we enumerate immediately
List <DataSourceValue> dataValues = QueryDataSourceValues(startTime, stopTime, interval, decimate, targetMap)
.TakeWhile(_ => !cancellationToken.IsCancellationRequested).ToList();
foreach (KeyValuePair <ulong, string> target in targetMap)
{
yield return new DataSourceValueGroup
{
Target = target.Value,
RootTarget = target.Value,
SourceTarget = sourceTarget,
Source = dataValues.Where(dataValue => dataValue.Target.Equals(target.Value)),
DropEmptySeries = dropEmptySeries
}
}
;
}
}
private Func <Guid, bool> BuildLookup(DataSet dataSource, Guid subscriberID)
{
HashSet <Guid> authorizedSignals = new HashSet <Guid>();
const string filterRegex = @"(ALLOW|DENY)\s+WHERE\s+([^;]*)";
DataRow subscriber;
DataRow[] subscriberMeasurementGroups;
//==================================================================
//Check if subscriber is disabled or removed
// If subscriber has been disabled or removed
// from the list of valid subscribers,
// they no longer have rights to any signals
subscriber = dataSource.Tables["Subscribers"].Select($"ID = '{subscriberID}' AND Enabled <> 0").FirstOrDefault();
if ((object)subscriber == null)
{
return(id => false);
}
//=================================================================
// Check group implicitly authorized signals
subscriberMeasurementGroups = dataSource.Tables["SubscriberMeasurementGroups"].Select($"SubscriberID = '{subscriberID}'");
subscriberMeasurementGroups
.Join(dataSource.Tables["MeasurementGroups"].Select(),
row => row.ConvertField <int>("MeasurementGroupID"),
row => row.ConvertField <int>("ID"),
(subscriberMeasurementGroup, measurementGroup) =>
{
bool allowed = subscriberMeasurementGroup.ConvertField <bool>("Allowed");
string filter = measurementGroup.ConvertField <string>("FilterExpression");
return(AdapterBase.ParseInputMeasurementKeys(dataSource, false, filter)
.Select(key => new { Allowed = allowed, key.SignalID }));
})
.SelectMany(list => list)
.GroupBy(obj => obj.SignalID)
.Where(grouping => grouping.All(obj => obj.Allowed))
.ToList()
.ForEach(grouping => authorizedSignals.Add(grouping.Key));
//=================================================================
//Check implicitly authorized signals
List <Match> matches = Regex.Matches(subscriber["AccessControlFilter"].ToNonNullString().ReplaceControlCharacters(), filterRegex, RegexOptions.IgnoreCase)
.Cast <Match>()
.ToList();
// Combine individual allow statements into a single measurement filter
string allowFilter = string.Join(" OR ", matches
.Where(match => match.Groups[1].Value == "ALLOW")
.Select(match => $"({match.Groups[2].Value})"));
// Combine individual deny statements into a single measurement filter
string denyFilter = string.Join(" OR ", matches
.Where(match => match.Groups[1].Value == "DENY")
.Select(match => $"({match.Groups[2].Value})"));
if (!string.IsNullOrEmpty(allowFilter))
{
foreach (DataRow row in dataSource.Tables["ActiveMeasurements"].Select(allowFilter))
{
authorizedSignals.Add(row.ConvertField <Guid>("SignalID"));
}
}
if (!string.IsNullOrEmpty(denyFilter))
{
foreach (DataRow row in dataSource.Tables["ActiveMeasurements"].Select(denyFilter))
{
authorizedSignals.Remove(row.ConvertField <Guid>("SignalID"));
}
}
//==================================================================
//Check explicit group authorizations
subscriberMeasurementGroups
.Join(dataSource.Tables["MeasurementGroupMeasurements"].Select(),
row => row.ConvertField <int>("MeasurementGroupID"),
row => row.ConvertField <int>("MeasurementGroupID"),
(subscriberMeasurementGroup, measurementGroupMeasurement) => new
{
Allowed = subscriberMeasurementGroup.ConvertField <bool>("Allowed"),
SignalID = measurementGroupMeasurement.ConvertField <Guid>("SignalID")
})
.GroupBy(obj => obj.SignalID)
.Select(grouping => new
{
Allowed = grouping.All(obj => obj.Allowed),
SignalID = grouping.Key
})
.ToList()
.ForEach(obj =>
{
if (obj.Allowed)
{
authorizedSignals.Add(obj.SignalID);
}
else
{
authorizedSignals.Remove(obj.SignalID);
}
});
//===================================================================
// Check explicit authorizations
DataRow[] explicitAuthorizations = dataSource.Tables["SubscriberMeasurements"].Select($"SubscriberID = '{subscriberID}'");
// Add all explicitly authorized signals to authorizedSignals
foreach (DataRow explicitAuthorization in explicitAuthorizations)
{
if (explicitAuthorization.ConvertField <bool>("Allowed"))
{
authorizedSignals.Add(explicitAuthorization.ConvertField <Guid>("SignalID"));
}
}
// Remove all explicitly unauthorized signals from authorizedSignals
foreach (DataRow explicitAthorization in explicitAuthorizations)
{
if (!explicitAthorization.ConvertField <bool>("Allowed"))
{
authorizedSignals.Remove(explicitAthorization.ConvertField <Guid>("SignalID"));
}
}
return(id => authorizedSignals.Contains(id));
}
// Internal Functions
private void ReadArchive(object state)
{
try
{
double timeRange = (m_settings.EndTime - m_settings.StartTime).TotalSeconds;
long receivedPoints = 0;
long processedDataBlocks = 0;
long duplicatePoints = 0;
Ticks operationTime;
Ticks operationStartTime;
DataPoint point = new DataPoint();
DateTime firstTimestamp = new DateTime(0L);
DateTime lastTimestamp = new DateTime(0L);
using (Algorithm algorithm = new Algorithm())
{
algorithm.ShowMessage = ShowUpdateMessage;
algorithm.MessageInterval = m_settings.MessageInterval;
algorithm.StartTime = m_settings.StartTime;
algorithm.EndTime = m_settings.EndTime;
algorithm.FrameRate = m_settings.FrameRate;
algorithm.TimeRange = timeRange;
algorithm.Log = m_log;
// Load historian meta-data
ShowUpdateMessage(">>> Loading source connection metadata...");
operationStartTime = DateTime.UtcNow.Ticks;
algorithm.Metadata = MetadataRecord.Query(m_settings.HostAddress, m_settings.MetadataPort, m_settings.MetadataTimeout);
operationTime = DateTime.UtcNow.Ticks - operationStartTime;
ShowUpdateMessage("*** Metadata Load Complete ***");
ShowUpdateMessage($"Total metadata load time {operationTime.ToElapsedTimeString(3)}...");
ShowUpdateMessage(">>> Processing filter expression for metadata...");
operationStartTime = DateTime.UtcNow.Ticks;
MeasurementKey[] inputKeys = AdapterBase.ParseInputMeasurementKeys(MetadataRecord.Metadata, false, textBoxPointList.Text, "MeasurementDetail");
List <ulong> pointIDList = inputKeys.Select(key => (ulong)key.ID).ToList();
operationTime = DateTime.UtcNow.Ticks - operationStartTime;
// Allow algorithm to augment (or even replace) point ID list as provided by user
algorithm.AugmentPointIDList(pointIDList);
ShowUpdateMessage($">>> Historian read will be for {pointIDList.Count:N0} points based on provided meta-data expression and algorithm augmentation.");
// Reduce metadata to filtered point list
ShowUpdateMessage($">>> Reducing metadata to the {pointIDList.Count:N0} defined points...");
List <MetadataRecord> records = new List <MetadataRecord>();
foreach (ulong pointID in pointIDList)
{
MetadataRecord record = algorithm.Metadata.FirstOrDefault(metadata => metadata.PointID == pointID);
if ((object)record != null)
{
records.Add(record);
}
}
algorithm.Metadata = records;
ShowUpdateMessage("*** Filter Expression Processing Complete ***");
ShowUpdateMessage($"Total filter expression processing time {operationTime.ToElapsedTimeString(3)}...");
ShowUpdateMessage(">>> Initializing algorithm...");
algorithm.Initialize();
ShowUpdateMessage(">>> Starting archive read...");
// Start historian data read
operationStartTime = DateTime.UtcNow.Ticks;
using (SnapDBClient historianClient = new SnapDBClient(m_settings.HostAddress, m_settings.DataPort, m_settings.InstanceName, m_settings.StartTime, m_settings.EndTime, m_settings.FrameRate, pointIDList))
{
// Scan to first record
if (!historianClient.ReadNext(point))
{
throw new InvalidOperationException("No data for specified time range in openHistorian connection!");
}
ulong currentTimestamp;
receivedPoints++;
while (!m_formClosing)
{
int timeComparison;
bool readSuccess = true;
// Create a new data block for current timestamp and load first/prior point
Dictionary <ulong, DataPoint> dataBlock = new Dictionary <ulong, DataPoint>
{
[point.PointID] = point.Clone()
};
currentTimestamp = point.Timestamp;
// Load remaining data for current timestamp
do
{
// Scan to next record
if (!historianClient.ReadNext(point))
{
readSuccess = false;
break;
}
receivedPoints++;
timeComparison = DataPoint.CompareTimestamps(point.Timestamp, currentTimestamp, m_settings.FrameRate);
if (timeComparison == 0)
{
// Timestamps are compared based on configured frame rate - if archived data rate is
// higher than configured frame rate, then data block will contain only latest values
if (dataBlock.ContainsKey(point.PointID))
{
duplicatePoints++;
}
dataBlock[point.PointID] = point.Clone();
}
}while (timeComparison == 0);
// Finished with data read
if (!readSuccess)
{
ShowUpdateMessage(">>> End of data in range encountered...");
break;
}
if (++processedDataBlocks % m_settings.MessageInterval == 0)
{
ShowUpdateMessage($"{Environment.NewLine}{receivedPoints:N0} points{(duplicatePoints > 0 ? $", which included {duplicatePoints:N0} duplicates," : "")} read so far averaging {receivedPoints / (DateTime.UtcNow.Ticks - operationStartTime).ToSeconds():N0} points per second.");
UpdateProgressBar((int)((1.0D - new Ticks(m_settings.EndTime.Ticks - (long)point.Timestamp).ToSeconds() / timeRange) * 100.0D));
}
try
{
lastTimestamp = new DateTime((long)currentTimestamp);
if (firstTimestamp.Ticks == 0L)
{
firstTimestamp = lastTimestamp;
}
// Analyze data block
algorithm.Execute(lastTimestamp, dataBlock.Values.ToArray());
}
catch (Exception ex)
{
ShowUpdateMessage($"ERROR: Algorithm exception: {ex.Message}");
m_log.Publish(MessageLevel.Error, "AlgorithmError", "Failed while processing data from the historian", exception: ex);
}
}
operationTime = DateTime.UtcNow.Ticks - operationStartTime;
if (m_formClosing)
{
ShowUpdateMessage("*** Historian Read Canceled ***");
UpdateProgressBar(0);
}
else
{
ShowUpdateMessage("*** Historian Read Complete ***");
UpdateProgressBar(100);
}
algorithm.Complete();
// Show some operational statistics
long expectedPoints = (long)(timeRange * m_settings.FrameRate * algorithm.Metadata.Count);
double dataCompleteness = Math.Round(receivedPoints / (double)expectedPoints * 100000.0D) / 1000.0D;
string overallSummary =
$"Total processing time {operationTime.ToElapsedTimeString(3)} at {receivedPoints / operationTime.ToSeconds():N0} points per second.{Environment.NewLine}" +
$"{Environment.NewLine}" +
$" Meta-data points: {algorithm.Metadata.Count}{Environment.NewLine}" +
$" Time-span covered: {timeRange:N0} seconds: {Ticks.FromSeconds(timeRange).ToElapsedTimeString(2)}{Environment.NewLine}" +
$" Processed timestamps: {processedDataBlocks:N0}{Environment.NewLine}" +
$" Expected points: {expectedPoints:N0} @ {m_settings.FrameRate:N0} samples per second{Environment.NewLine}" +
$" Received points: {receivedPoints:N0}{Environment.NewLine}" +
$" Duplicate points: {duplicatePoints:N0}{Environment.NewLine}" +
$" Data completeness: {dataCompleteness:N3}%{Environment.NewLine}" +
$" First timestamp with data: {firstTimestamp:yyyy-MM-dd HH:mm:ss.fff}{Environment.NewLine}" +
$" Last timestamp with data: {lastTimestamp:yyyy-MM-dd HH:mm:ss.fff}{Environment.NewLine}";
ShowUpdateMessage(overallSummary);
}
}
}
catch (Exception ex)
{
ShowUpdateMessage($"!!! Failure during historian read: {ex.Message}");
m_log.Publish(MessageLevel.Error, "HistorianDataRead", "Failed while reading data from the historian", exception: ex);
}
finally
{
SetGoButtonEnabledState(true);
}
}
