private async Task RunBoardControlLoops(List <IOconfOut230Vac> ports, CancellationToken token)
{
DateTime start = DateTime.Now;
try
{
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, _boardLoopsStopTokenSource.Token);
//we ignore remote boards and boards missing during the start sequence (as we don't have auto reconnect logic yet for those). Note the reader in the ctor already reports the missing local boards.
var connectedBoards = ports.Where(p => p.Map.IsLocalBoard && p.Map.Board != null);
var boardLoops = connectedBoards
.GroupBy(v => v.Map.Board)
.Select(g => BoardLoop(g.Key, g.ToList(), linkedCts.Token))
.ToList();
await Task.WhenAll(boardLoops);
CALog.LogInfoAndConsoleLn(LogID.A, "Exiting SwitchBoardController.RunBoardControlLoops() " + DateTime.Now.Subtract(start).Humanize(5));
}
catch (Exception ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, ex.ToString());
}
finally
{
_boardControlLoopsStopped.TrySetResult();
}
}
private async Task BoardLoop(MCUBoard board, List <IOconfOut230Vac> ports, CancellationToken token)
{
var lastActions = new SwitchboardAction[ports.Max(p => p.PortNumber)];
var boardStateName = board.BoxName + "_state";
// we use the next 2 booleans to avoid spamming logs/display with an ongoing problem, so we only notify at the beginning and when we resume normal operation.
// we might still get lots of entries for problems that alternate between normal and failed states, but for now is a good data point to know if that case is happening.
var waitingBoardReconnect = false;
var tryingToRecoverAfterTimeoutWatch = new Stopwatch();
while (!token.IsCancellationRequested)
{
try
{
var vector = await _cmd.When(_ => true, token);
if (!CheckConnectedStateInVector(board, boardStateName, ref waitingBoardReconnect, vector))
{
continue; // no point trying to send commands while there is no connection to the board.
}
foreach (var port in ports)
{
await DoPortActions(vector, board, port, lastActions, token);
}
if (tryingToRecoverAfterTimeoutWatch.IsRunning)
{
tryingToRecoverAfterTimeoutWatch.Stop();
CALog.LogInfoAndConsoleLn(LogID.A, $"wrote to switch board without time outs after {tryingToRecoverAfterTimeoutWatch.Elapsed}, resuming normal action frequency - {board.ToShortDescription()}");
}
}
catch (TimeoutException)
{
if (!tryingToRecoverAfterTimeoutWatch.IsRunning)
{
// we only notify of the situation once while this is a different way a disconnect might look like, we have
CALog.LogInfoAndConsoleLn(LogID.A, $"timed out writing to switchboard, reducing action frequency until reconnect - {board.ToShortDescription()}");
tryingToRecoverAfterTimeoutWatch.Restart();
}
// forcing reduced acting frequency )
try { await Task.Delay(500, token); }
catch (TaskCanceledException) { }
}
catch (TaskCanceledException ex)
{
if (!token.IsCancellationRequested)
{
CALog.LogErrorAndConsoleLn(LogID.A, ex.ToString());
}
}
catch (Exception ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, ex.ToString());
}
}
AllOff(board, ports);
}
private static bool CheckConnectedStateInVector(MCUBoard board, string boardStateName, ref bool waitingBoardReconnect, NewVectorReceivedArgs vector)
{
var vectorState = (BaseSensorBox.ConnectionState)(int) vector[boardStateName];
var connected = vectorState >= BaseSensorBox.ConnectionState.Connected;
if (waitingBoardReconnect && connected)
{
CALog.LogData(LogID.B, $"resuming switchboard actions after reconnect on {board.ToShortDescription()}");
waitingBoardReconnect = false;
}
else if (!waitingBoardReconnect && !connected)
{
CALog.LogData(LogID.B, $"stopping switchboard actions while connection is reestablished - state: {vectorState} - {board.ToShortDescription()}");
waitingBoardReconnect = true;
}
return(connected);
}
private static void AllOff(MCUBoard board, List <IOconfOut230Vac> ports)
{
try
{
board.SafeWriteLine("off", CancellationToken.None);
foreach (var port in ports)
{
if (port.HasOnSafeState)
{
board.SafeWriteLine($"p{port.PortNumber} on", CancellationToken.None);
}
}
}
catch (Exception ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, $"Error detected while attempting to set ports to default positions for board {board.ToShortDescription()}", ex);
}
}
private static async Task <MCUBoard> AttemptToOpenDeviceConnection(string name, CALogLevel logLevel)
{
try
{
var mcu = await MCUBoard.OpenDeviceConnection(name);
mcu.productType = mcu.productType ?? GetStringFromDmesg(mcu.PortName);
string logline = logLevel == CALogLevel.Debug ? mcu.ToDebugString(Environment.NewLine) : mcu.ToString();
CALog.LogInfoAndConsoleLn(LogID.A, logline);
return(mcu);
}
catch (UnauthorizedAccessException ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, $"Unable to open {name}, Exception: {ex.Message}" + Environment.NewLine);
}
catch (Exception ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, $"Unable to open {name}, Exception: {ex}" + Environment.NewLine);
}
return(default);
private void LoopForever()
{
_running = true;
var vectorUploadDelay = IOconfFile.GetVectorUploadDelay();
var throttle = new TimeThrottle(vectorUploadDelay);
while (_running)
{
throttle.Wait();
try
{
var list = DequeueAllEntries(_queue);
if (list != null)
{
PostVectorAsync(GetSignedVectors(list), list.First().timestamp);
}
var events = DequeueAllEntries(_eventsQueue);
if (events != null)
{
foreach (var @event in events)
{
PostEventAsync(@event);
}
}
PrintBadPackagesMessage(false);
}
catch (Exception ex)
{
if (!Environment.MachineName.StartsWith("DESKTOP"))
{
CALog.LogErrorAndConsoleLn(LogID.A, "ServerUploader.LoopForever() exception: " + ex.Message, ex);
}
}
}
PrintBadPackagesMessage(true);
}
private static async Task DoPortActions(NewVectorReceivedArgs vector, MCUBoard board, IOconfOut230Vac port, SwitchboardAction[] lastActions, CancellationToken token)
{
if (token.IsCancellationRequested)
{
return;
}
try
{
var action = SwitchboardAction.FromVectorSamples(vector, port.Name);
if (action.Equals(lastActions[port.PortNumber - 1]))
{
return; // no action changes has been requested since the last action taken on the heater.
}
var onSeconds = action.GetRemainingOnSeconds(vector.GetVectorTime());
if (onSeconds <= 0)
{
await board.SafeWriteLine($"p{port.PortNumber} off", token);
}
else if (onSeconds == int.MaxValue)
{
await board.SafeWriteLine($"p{port.PortNumber} on", token);
}
else
{
await board.SafeWriteLine($"p{port.PortNumber} on {onSeconds}", token);
}
lastActions[port.PortNumber - 1] = action;
}
catch (TimeoutException)
{
// we don't want logging at this level as the caller handles this in a way that reduces the amount of noise for failures that last many vectors.
throw;
}
catch (Exception)
{
CALog.LogErrorAndConsoleLn(LogID.A, $"Failed executing port action for board {board.BoxName} port {port.Name} - {port.Name}");
throw; // this will log extra info and avoid extra board actions on this cycle
}
}
public void SendVector(List <double> vector, DateTime timestamp)
{
if (vector.Count() != _vectorDescription.Length)
{
throw new ArgumentException($"wrong vector length (input, expected): {vector.Count} <> {_vectorDescription.Length}");
}
if (timestamp <= _lastTimestamp)
{
CALog.LogData(LogID.B, $"non changing or out of order timestamp received - vector ignored: last recorded {_lastTimestamp} vs received {timestamp}");
return;
}
lock (_queue)
if (_queue.Count < 10000) // if sending thread can't catch up, then drop packages.
{
_queue.Enqueue(new DataVector
{
timestamp = timestamp,
vector = vector
});
_lastTimestamp = timestamp;
}
}
private static void OnError(string message, Exception ex) => CALog.LogErrorAndConsoleLn(LogID.A, message, ex);
private async static Task <MCUBoard> Create(string name, int baudrate, bool skipBoardAutoDetection)
{
MCUBoard board = null;
try
{
var port = new SerialPort(name);
board = new MCUBoard(port);
port.BaudRate = 1;
port.DtrEnable = true;
port.RtsEnable = true;
port.BaudRate = baudrate;
port.PortName = name;
port.ReadTimeout = 2000;
port.WriteTimeout = 2000;
port.Open();
board.pipeReader = PipeReader.Create(port.BaseStream);
Thread.Sleep(30); // it needs to await that the board registers that the COM port has been opened before sending commands (work arounds issue when first opening the connection and sending serial).
board.TryReadLine = board.TryReadAsciiLine; //this is the default which can be changed in ReadSerial based on the ThirdPartyProtocolDetection
board.productType = "NA";
if (skipBoardAutoDetection)
{
board.InitialConnectionSucceeded = true;
}
else
{
board.InitialConnectionSucceeded = await board.ReadSerialNumber();
}
if (File.Exists("IO.conf"))
{ // note that unlike the discovery at MCUBoard.OpenDeviceConnection that only considers the usb port, in here we can find the boards by the serial number too.
foreach (var ioconfMap in IOconfFile.GetMap())
{
if (ioconfMap.SetMCUboard(board))
{
board.BoxName = ioconfMap.BoxName;
board.ConfigSettings = ioconfMap.BoardSettings;
port.ReadTimeout = ioconfMap.BoardSettings.MaxMillisecondsWithoutNewValues;
await board.UpdateCalibration(board.ConfigSettings);
}
}
}
}
catch (Exception ex)
{
CALog.LogException(LogID.A, ex);
}
if (board != null && !board.InitialConnectionSucceeded)
{
board.pipeReader?.Complete();
board.port.Close();
Thread.Sleep(100);
}
else if (board != null && board.IsEmpty() && board.BoxName == null)
{//note we don't log this for devices without serial that were mapped by usb (last check above)
CALog.LogInfoAndConsoleLn(LogID.B, $"some data without serial detected for device at port {name} - {baudrate} / still connected");
}
return(board);
}
public void LogData(string message) => CALog.LogData(LogID.B, FormatMessage(message));
public void LogInfo(string message) => CALog.LogInfoAndConsoleLn(LogID.A, FormatMessage(message));
public void LogError(Exception ex) => CALog.LogException(LogID.A, ex);
private void WaitForLoopStopped(object sender, EventArgs args)
{
CALog.LogData(LogID.A, "waiting for switchboards control loops to finish actions and set ports to their default value");
_boardControlLoopsStopped.Task.Wait();
CALog.LogData(LogID.A, "finished waiting for switchboards loops");
}