// Static Methods
private static bool TryFindPIPoint(PIConnection connection, DataSet metadata, string pointTag, out PIPoint point)
{
DataRow[] rows = metadata.Tables["ActiveMeasurements"].Select($"PointTag = '{pointTag}'");
if (rows.Length <= 0)
{
point = null;
return false;
}
return TryFindPIPoint(connection, pointTag, rows[0]["AlternateTag"].ToString(), out point);
}
public void Run(string server, string pointsQuery)
{
var connection = new PIConnection(server);
_mainTask = Task.Run(() =>
{
WaitForServerToBeAvailable(connection);
var pointsProvier = new PIPointsProvider(pointsQuery, connection.GetPiServer());
_dataReader = new DataReader(pointsProvier, _queue, _cancellationTokenSource.Token);
_dataWriter = new DataWriter(_queue, connection.GetPiServer());
_dataWriter.Run();
_dataReader.RunBackfill();
_logger.Info("Starting the normal operations process");
_dataReader.Run(General.Default.NormalDataCollectionFrequencySeconds);
}, _cancellationTokenSource.Token);
}
private void WaitForServerToBeAvailable(PIConnection connection)
{
var isConnected = false;
while (!isConnected)
{
isConnected = connection.Connect();
if (!isConnected)
{
_logger.InfoFormat("Could not connect to the server, waiting 5s to try to reconnect.");
Thread.Sleep(TimeSpan.FromSeconds(5));
}
if (_cancellationTokenSource.Token.IsCancellationRequested)
{
_logger.InfoFormat("Cancellation requested, exiting WaitForServerToBeAvailable");
return;
}
}
}
/// <summary>
/// Releases the unmanaged resources used by the <see cref="PIRTInputAdapter"/> object and optionally releases the managed resources.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (!m_disposed)
{
try
{
if (disposing)
{
if ((object)m_connection != null)
{
m_connection.Disconnected -= m_connection_Disconnected;
m_connection.Dispose();
m_connection = null;
}
}
}
finally
{
m_disposed = true; // Prevent duplicate dispose.
base.Dispose(disposing); // Call base class Dispose().
}
}
}
private static void Main(string[] args)
{
TextWriter writer = Console.Out;
try
{
var options = new CommandLineOptions();
if (Parser.Default.ParseArguments(args, options))
{
if (args.Length <= 1)
{
Console.Write(options.GetUsage());
}
if (options.Run != null && ValidateRunOptions(options.Run))
{
_logger.Info("Option Run starting, will run the data replay continuously as a command line application.");
var replayer = new Replayer();
replayer.RunFromCommandLine(options.Run[0], options.Run[1]);
}
if (options.deleteHistory != null && ValidateDeleteHistoryOptions(options.deleteHistory))
{
_logger.Info("Delete History Option Selected, will deleted the specified data.");
_logger.Info("This operation cannot be reversed, are you sure you want to delete the data you specified? Press Y to continue...");
var keyInfo = Console.ReadKey();
if (keyInfo.KeyChar != 'Y')
{
_logger.Info("Operation canceled");
}
// getting the tags
var piConnection = new PIConnection(options.deleteHistory[0]);
piConnection.Connect();
var pointsProvider = new PIPointsProvider(options.deleteHistory[3], piConnection.GetPiServer());
foreach (var piPoint in pointsProvider.Points)
{
var st = AFTime.Parse(options.deleteHistory[1]);
var et = AFTime.Parse(options.deleteHistory[2]);
_logger.InfoFormat("Deleting history for tag: {0} between {1:G} and {2:G}", piPoint.Name, st.LocalTime, et.LocalTime);
PIHelpers.DeleteValues(piPoint, st, et, options.ForceUpdateValuesMethod);
}
}
}
else
{
options.GetUsage();
}
}
catch (Exception ex)
{
Console.SetOut(writer);
Console.WriteLine("Error: " + ex);
}
}
private static void Main(string[] args)
{
PIConnection piConnection;
var _logger = LogManager.GetLogger(typeof(Program));
try
{
var options = new CommandLineOptions();
if (Parser.Default.ParseArguments(args, options))
{
ValidateSettings(options);
var readerSettings = new DataReaderSettings();
if (options.Server.Length == 1)
{
piConnection = new PIConnection(options.Server[0]);
}
else
{
piConnection = new PIConnection(options.Server[0], options.Server[1]);
}
if (options.testTagSearch != null && options.testTagSearch.Length > 0)
{
_logger.Info("Search test started...");
piConnection.Connect();
var search = new TagsLoader(piConnection.GetPiServer());
foreach (var s in options.testTagSearch)
{
var tags = search.Search(s).ToList();
_logger.WarnFormat("Found {0} tags with query {1}", tags.Count, s);
if (options.testTagSearchPrintAllTags)
{
tags.ForEach(t => _logger.InfoFormat("Tag: {0}, PointClass: {1}", t.Name, t.PointClass));
}
}
}
if (options.TagQueries != null && options.TagQueries.Length > 0)
{
_logger.Info("Data reader starting...");
piConnection.Connect();
if (options.EventsPerDay > 0 && options.TagsCount > 0)
{
// var type = options.UseParallel? DataReaderSettings.ReadingType.Parallel: DataReaderSettings.ReadingType.Bulk;
var type = DataReaderSettings.ReadingType.Bulk;
readerSettings.AutoTune(type, options.EventsPerDay, options.TagsCount, options.EventsPerRead);
}
// starts the data writer
// settings data filters to filter out the data if option is specified
var filtersFactory = new FiltersFactory();
if (options.RemoveDuplicates)
{
filtersFactory.AddFilter(new DuplicateValuesFilter());
}
if (options.FilterDigitalStates)
{
filtersFactory.AddFilter(new SystemStatesFilter());
}
_logger.Info("Creating worker objects...");
var dataWriter = new DataWriter(options.OutfileName, options.EventsPerFile, options.WritersCount, filtersFactory);
var dataReader = new DataReaderBulk(readerSettings, dataWriter, options.EnableWrite);
//dataReader = options.UseParallel
// ? (IDataReader) new DataReaderParallel(readerSettings, dataWriter)
// : new DataReaderBulk(readerSettings, dataWriter);
var orchestrator = new Orchestrator(options.StartTime, options.EndTime,
readerSettings.TimeIntervalPerDataRequest, dataReader);
var tagsLoader = new TagsLoader(piConnection.GetPiServer(), options.TagQueries,
readerSettings.TagGroupSize, orchestrator);
var statistics = new Statistics();
// starts the orchestrator
_logger.Info("Starting workers...");
var tagsLoaderTask = tagsLoader.Run();
var writerTask = dataWriter.Run();
// var processorTask = dataProcessor.Run(); -- not using this after determining this was not bringing much performance gain.
var orchestratorTask = orchestrator.Run();
var dataReaderTask = dataReader.Run();
var statsTask = statistics.Run();
// starts the data reader
Task.WaitAll(orchestratorTask, writerTask, dataReaderTask, tagsLoaderTask);
statistics.Stop();
Task.WaitAll(statsTask);
_logger.Info("All tasks completed successfully");
}
// DEBUG
// Console.ReadKey();
// exit ok
Environment.Exit(0);
}
else
{
// exit with error
Environment.Exit(1);
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
}
/// <summary>
/// Closes this <see cref="PIOutputAdapter"/> connections to the PI server.
/// </summary>
protected override void AttemptDisconnection()
{
foreach (ProcessQueue<AFValue> archiveQueue in m_archiveQueues)
archiveQueue.Stop();
m_mapRequestQueue.Stop();
m_mapRequestQueue.Clear();
m_mappedPIPoints.Clear();
lock (m_pendingMappings)
{
m_pendingMappings.Clear();
}
if ((object)m_connection != null)
{
m_connection.Disconnected -= m_connection_Disconnected;
m_connection.Dispose();
m_connection = null;
}
}
/// <summary>
/// Connects to the configured PI server.
/// </summary>
protected override void AttemptConnection()
{
m_processedMappings = 0;
m_processedMeasurements = 0;
m_totalProcessingTime = 0;
m_connection = new PIConnection
{
ServerName = this.ServerName,
UserName = this.UserName,
Password = this.Password,
ConnectTimeout = this.ConnectTimeout
};
m_connection.Disconnected += m_connection_Disconnected;
m_connection.Open();
m_mappedPIPoints.Clear();
lock (m_pendingMappings)
{
m_pendingMappings.Clear();
}
m_mapRequestQueue.Clear();
m_mapRequestQueue.Start();
foreach (ProcessQueue<AFValue> archiveQueue in m_archiveQueues)
archiveQueue.Start();
// Kick off meta-data refresh
RefreshMetadata();
}
/// <summary>
/// Attempts to connect to this <see cref="PIAdapters.PIPBInputAdapter"/>.
/// </summary>
protected override void AttemptConnection()
{
// This adapter is only engaged for history, so we don't process any data unless a temporal constraint is defined
if (this.TemporalConstraintIsDefined())
{
// Turn off read timer if it's active
m_readTimer.Enabled = false;
m_connection = new PIConnection
{
ServerName = m_serverName,
UserName = m_userName,
Password = m_password,
ConnectTimeout = m_connectTimeout
};
m_connection.Open();
// Start the data reader on its own thread so connection attempt can complete in a timely fashion...
ThreadPool.QueueUserWorkItem(StartDataReader);
}
}
/// <summary>
/// Gets a connection from the pool or creates a new one if all current connections are being used at peak access.
/// </summary>
/// <param name="serverName">Name of the PI server for the adapter's PI connection.</param>
/// <param name="userName">Name of the PI user ID for the adapter's PI connection.</param>
/// <param name="password">Password used for the adapter's PI connection.</param>
/// <param name="connectTimeout">Timeout interval (in milliseconds) for the adapter's connection.</param>
/// <returns>A <see cref="PIConnection"/> from the pool.</returns>
/// <exception cref="InvalidOperationException">Failed to get a pooled PI connection.</exception>
public PIConnection GetPooledConnection(string serverName, string userName = null, string password = null, int connectTimeout = PIConnection.DefaultConnectTimeout)
{
PIConnection connection = null;
// We dynamically allocate pooled PI server connections each having a maximum accessibility count.
// PI's threading model can handle many connections each archiving a small volume of points, but
// falls behind under load when archiving a large volume of points from a single connection.
try
{
lock (m_connectionPool)
{
while ((object)connection == null)
{
// Get next connection from the pool with lowest accessibility count
if (m_connectionPool.Count > 0)
{
PIConnection[] availableConnections = m_connectionPool.Where(c => c.AccessCount < m_accessCountPerConnection).ToArray();
if (availableConnections.Length > 0)
connection = availableConnections.Aggregate((currentMin, nextItem) => (object)nextItem != null && currentMin.AccessCount < nextItem.AccessCount ? currentMin : nextItem);
}
if ((object)connection == null)
{
// Add pooled connections in groups for better distribution
for (int i = 0; i < m_minimumPoolSize; i++)
{
// Create a new connection
connection = new PIConnection
{
ServerName = serverName,
UserName = userName,
Password = password,
ConnectTimeout = connectTimeout
};
// Since PI doesn't detect disconnection until an operation is attempted,
// we must monitor for disconnections from the pooled connections as well
connection.Disconnected += connection_Disconnected;
connection.Open();
// Add the new connection to the server pool
m_connectionPool.Add(connection);
}
}
}
// Increment current connection access count
connection.AccessCount++;
}
}
catch (Exception ex)
{
throw new InvalidOperationException(string.Format("Failed to get a pooled PI connection: {0}", ex.Message), ex);
}
return connection;
}
private static bool TryFindPIPoint(PIConnection connection, string pointTag, string alternateTag, out PIPoint point)
{
return PIPoint.TryFindPIPoint(connection.Server, string.IsNullOrWhiteSpace(alternateTag) ? pointTag : alternateTag, out point);
}
// TODO: Update code to properly handle piped-events in AF-SDK
//private void PipeOnOnNewValue()
//{
// List<IMeasurement> measurements = new List<IMeasurement>();
// m_connection.Execute(server =>
// {
// PIEventObject eventobject;
// PointValue pointvalue;
// for (int i = 0; i < m_pipe.Count; i++)
// {
// eventobject = m_pipe.Take();
// // we will publish measurements for every action except deleted (possible dupes on updates)
// if (eventobject.Action != EventActionConstants.eaDelete)
// {
// try
// {
// pointvalue = (PointValue)eventobject.EventData;
// double value = Convert.ToDouble(pointvalue.PIValue.Value);
// MeasurementKey key = m_tagKeyMap[pointvalue.PIPoint.Name];
// Measurement measurement = new Measurement();
// measurement.Key = key;
// measurement.Timestamp = pointvalue.PIValue.TimeStamp.LocalDate.ToUniversalTime();
// measurement.Value = value;
// measurement.StateFlags = MeasurementStateFlags.Normal;
// if (measurement.Timestamp > m_lastReceivedTimestamp.Ticks)
// m_lastReceivedTimestamp = measurement.Timestamp;
// measurements.Add(measurement);
// }
// catch
// {
// /* squelch any errors on digital state data that can't be converted to a double */
// }
// }
// }
// });
// if (measurements.Any())
// {
// OnNewMeasurements(measurements);
// m_processedMeasurements += measurements.Count;
// }
//}
/// <summary>
/// Disposes members for garbage collection
/// </summary>
/// <param name="disposing"></param>
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
if (m_tagKeyMap != null)
m_tagKeyMap.Clear();
m_tagKeyMap = null;
if ((object)m_connection != null)
m_connection.Dispose();
m_connection = null;
m_points = null;
//m_pipe = null;
m_measurements.Clear();
m_measurements = null;
if (m_dataThread != null)
{
m_dataThread.Abort();
m_dataThread = null;
}
if (m_publishTimer != null)
{
m_publishTimer.Stop();
m_publishTimer.Elapsed -= m_publishTimer_Tick;
m_publishTimer.Dispose();
m_publishTimer = null;
}
}
/// <summary>
/// Disconnects from the configured PI server if a connection is open
/// </summary>
protected override void AttemptDisconnection()
{
if ((object)m_connection != null)
{
m_connection.Dispose();
m_connection = null;
}
}
/// <summary>
/// Connects to the configured PI server.
/// </summary>
protected override void AttemptConnection()
{
m_processedMeasurements = 0;
m_connection = new PIConnection
{
ServerName = m_serverName,
UserName = m_userName,
Password = m_password,
ConnectTimeout = m_connectTimeout
};
m_connection.Open();
}
/// <summary>
/// Attempts to disconnect from this <see cref="PIAdapters.PIPBInputAdapter"/>.
/// </summary>
protected override void AttemptDisconnection()
{
if ((object)m_readTimer != null)
{
m_readTimer.Enabled = false;
lock (m_readTimer)
{
m_dataReader = null;
}
}
if ((object)m_connection != null)
{
m_connection.Dispose();
m_connection = null;
}
}
/// <summary>
/// Connects to the configured PI server.
/// </summary>
protected override void AttemptConnection()
{
m_connection = new PIConnection
{
ServerName = this.ServerName,
UserName = this.UserName,
Password = this.Password,
ConnectTimeout = this.ConnectTimeout
};
m_connection.Disconnected += m_connection_Disconnected;
m_connection.Open();
m_dataPipe = new PIDataPipe(AFDataPipeType.Snapshot);
//m_dataPipe.Subscribe(m_dataUpdateObserver);
if (AutoStart && (object)OutputMeasurements != null && OutputMeasurements.Any())
SubscribeToPointUpdates(this.OutputMeasurementKeys());
}
/// <summary>
/// Disconnects from the configured PI server if a connection is open
/// </summary>
protected override void AttemptDisconnection()
{
m_eventTimer.Enabled = false;
if ((object)m_dataPoints != null)
{
m_dataPoints.Clear();
m_dataPoints = null;
}
if ((object)m_dataPipe != null)
{
m_dataPipe.Dispose();
m_dataPipe = null;
}
if ((object)m_connection != null)
{
m_connection.Dispose();
m_connection = null;
}
}
/// <summary>
/// Releases the unmanaged resources used by the <see cref="PIOutputAdapter"/> object and optionally releases the managed resources.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (!m_disposed)
{
try
{
if (disposing)
{
if ((object)m_mapRequestQueue != null)
m_mapRequestQueue.Dispose();
if ((object)m_archiveQueues != null)
{
foreach (ProcessQueue<AFValue> archiveQueue in m_archiveQueues)
archiveQueue.Dispose();
}
if ((object)m_connection != null)
{
m_connection.Disconnected -= m_connection_Disconnected;
m_connection.Dispose();
m_connection = null;
}
if ((object)m_mappedPIPoints != null)
m_mappedPIPoints.Clear();
}
}
finally
{
m_disposed = true; // Prevent duplicate dispose.
base.Dispose(disposing); // Call base class Dispose().
}
}
}
/// <summary>
/// Returns <see cref="PIConnection"/> to the pool.
/// </summary>
/// <param name="connection"><see cref="PIConnection"/> to return to the pool.</param>
/// <exception cref="InvalidOperationException">Provided PIConnection does not belong to this connection pool.</exception>
public void ReturnPooledConnection(PIConnection connection)
{
if ((object)connection == null)
return;
lock (m_connectionPool)
{
if (!m_connectionPool.Contains(connection))
throw new InvalidOperationException("Provided PIConnection does not belong to this connection pool");
if (connection.AccessCount > 0)
connection.AccessCount--;
if (connection.AccessCount < 1)
{
if (m_connectionPool.Count > m_minimumPoolSize && m_connectionPool.Remove(connection))
{
connection.Disconnected -= connection_Disconnected;
connection.Dispose();
}
}
}
}
private static void Main(string[] args)
{
PIConnection piConnection;
var _logger = LogManager.GetLogger(typeof (Program));
try
{
var options = new CommandLineOptions();
if (Parser.Default.ParseArguments(args, options))
{
ValidateSettings(options);
var readerSettings = new DataReaderSettings();
if(options.Server.Length==1)
piConnection = new PIConnection(options.Server[0]);
else
piConnection = new PIConnection(options.Server[0], options.Server[1]);
if (options.testTagSearch != null && options.testTagSearch.Length > 0)
{
_logger.Info("Search test started...");
piConnection.Connect();
var search = new TagsLoader(piConnection.GetPiServer());
foreach (var s in options.testTagSearch)
{
var tags = search.Search(s).ToList();
_logger.WarnFormat("Found {0} tags with query {1}", tags.Count, s);
if (options.testTagSearchPrintAllTags)
{
tags.ForEach(t => _logger.InfoFormat("Tag: {0}, PointClass: {1}", t.Name, t.PointClass));
}
}
}
if (options.TagQueries != null && options.TagQueries.Length > 0)
{
_logger.Info("Data reader starting...");
piConnection.Connect();
if (options.EventsPerDay > 0 && options.TagsCount > 0)
{
// var type = options.UseParallel? DataReaderSettings.ReadingType.Parallel: DataReaderSettings.ReadingType.Bulk;
var type = DataReaderSettings.ReadingType.Bulk;
readerSettings.AutoTune(type, options.EventsPerDay, options.TagsCount, options.EventsPerRead);
}
// starts the data writer
// defines rejected states to filter out
var rejectedStates = new[]
{
"pt created",
"snapfix",
"shutdown",
"no data",
"bad",
"No Alarm",
"High Alarm",
"Low Alarm",
"Hi Alarm/Ack",
"Lo Alarm/Ack",
"NoAlrm/UnAck",
"Bad Quality",
"Rate Alarm",
"Rate Alm/Ack",
"Dig Alarm",
"Dig Alm/Ack",
"?204",
"?205",
"?206",
"?207",
"?208",
"?209",
"AccessDenied",
"No Sample",
"No Result",
"Unit Down",
"Sample Bad",
"Equip Fail",
"No Lab Data",
"Trace",
"GreaterMM",
"Bad Lab Data",
"Good-Off",
"Good-On",
"Alarm-Off",
"Alarm-On",
"Bad_Quality",
"BadQ-On",
"BadQ-Alrm-Of",
"BadQ-Alrm-On",
"?228",
"?229",
"Manual",
"Auto",
"Casc/Ratio",
"DCS failed",
"Manual Lock",
"CO Bypassed",
"?236",
"Bad Output",
"Scan Off",
"Scan On",
"Configure",
"Failed",
"Error",
"Execute",
"Filtered",
"Calc Off",
"I/O Timeout",
"Set to Bad",
"Calc Failed",
"Calc Overflw",
"Under Range",
"Over Range",
"Bad Input",
"Bad Total",
"No_Alarm",
"Over UCL",
"Under LCL",
"Over WL",
"Under WL",
"Over 1 Sigma",
"Under 1Sigma",
"Over Center",
"Under Center",
"Stratified",
"Mixture",
"Trend Up",
"Trend Down",
"No Alarm#",
"Over UCL#",
"Under LCL#",
"Over WL#",
"Under WL#",
"Over 1Sigma#",
"Under 1Sigm#",
"Over Center#",
"Under Centr#",
"Stratified#",
"Mixture#",
"Trend Up#",
"Trend Down#",
"?283",
"?284",
"?285",
"?286",
"?287",
"?288",
"ActiveBatch",
"Bad Data",
"Calc Crash",
"Calc Timeout",
"Bad Narg",
"Inp OutRange",
"Not Converge",
"DST Forward",
"DST Back",
"Substituted",
"Invalid Data",
"Scan Timeout",
"No_Sample",
"Arc Off-line",
"ISU Saw No Data",
"-err",
"Good",
"_SUBStituted",
"Doubtful",
"Wrong Type",
"Overflow_st",
"Intf Shut",
"Out of Serv",
"Comm Fail",
"Not Connect",
"Coercion Failed",
"Invalid Float",
"Future Data Unsupported"
};
var filtersFactory=new FiltersFactory();
filtersFactory.SetDigitalStatesFilters(rejectedStates);
filtersFactory.SetFilters(FiltersFactory.FiltersTypesEnum.DigitalStatesFilter,FiltersFactory.FiltersTypesEnum.DuplicateValuesFilter);
_logger.Info("Creating worker objects...");
var dataWriter = new DataWriter(options.OutfileName, options.EventsPerFile, options.WritersCount, filtersFactory);
var dataReader = new DataReaderBulk(readerSettings, dataWriter, options.EnableWrite);
//dataReader = options.UseParallel
// ? (IDataReader) new DataReaderParallel(readerSettings, dataWriter)
// : new DataReaderBulk(readerSettings, dataWriter);
var orchestrator = new Orchestrator(options.StartTime, options.EndTime,
readerSettings.TimeIntervalPerDataRequest, dataReader);
var tagsLoader = new TagsLoader(piConnection.GetPiServer(), options.TagQueries,
readerSettings.TagGroupSize, orchestrator);
var statistics = new Statistics();
// starts the orchestrator
_logger.Info("Starting workers...");
var tagsLoaderTask = tagsLoader.Run();
var writerTask = dataWriter.Run();
// var processorTask = dataProcessor.Run();
var orchestratorTask = orchestrator.Run();
var dataReaderTask = dataReader.Run();
var statsTask = statistics.Run();
// starts the data reader
Task.WaitAll(orchestratorTask, writerTask, dataReaderTask, tagsLoaderTask);
statistics.Stop();
Task.WaitAll(statsTask);
_logger.Info("All tasks completed successfully");
}
// DEBUG
// Console.ReadKey();
// exit ok
Environment.Exit(0);
}
else
{
// exit with error
Environment.Exit(1);
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
}
