private async Task RunBoardControlLoops(List <IOconfOut230Vac> ports, CancellationToken token)
{
DateTime start = DateTime.Now;
try
{
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, _boardLoopsStopTokenSource.Token);
//we ignore remote boards and boards missing during the start sequence (as we don't have auto reconnect logic yet for those). Note the reader in the ctor already reports the missing local boards.
var connectedBoards = ports.Where(p => p.Map.IsLocalBoard && p.Map.Board != null);
var boardLoops = connectedBoards
.GroupBy(v => v.Map.Board)
.Select(g => BoardLoop(g.Key, g.ToList(), linkedCts.Token))
.ToList();
await Task.WhenAll(boardLoops);
CALog.LogInfoAndConsoleLn(LogID.A, "Exiting SwitchBoardController.RunBoardControlLoops() " + DateTime.Now.Subtract(start).Humanize(5));
}
catch (Exception ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, ex.ToString());
}
finally
{
_boardControlLoopsStopped.TrySetResult();
}
}
private async Task BoardLoop(MCUBoard board, List <IOconfOut230Vac> ports, CancellationToken token)
{
var lastActions = new SwitchboardAction[ports.Max(p => p.PortNumber)];
var boardStateName = board.BoxName + "_state";
// we use the next 2 booleans to avoid spamming logs/display with an ongoing problem, so we only notify at the beginning and when we resume normal operation.
// we might still get lots of entries for problems that alternate between normal and failed states, but for now is a good data point to know if that case is happening.
var waitingBoardReconnect = false;
var tryingToRecoverAfterTimeoutWatch = new Stopwatch();
while (!token.IsCancellationRequested)
{
try
{
var vector = await _cmd.When(_ => true, token);
if (!CheckConnectedStateInVector(board, boardStateName, ref waitingBoardReconnect, vector))
{
continue; // no point trying to send commands while there is no connection to the board.
}
foreach (var port in ports)
{
await DoPortActions(vector, board, port, lastActions, token);
}
if (tryingToRecoverAfterTimeoutWatch.IsRunning)
{
tryingToRecoverAfterTimeoutWatch.Stop();
CALog.LogInfoAndConsoleLn(LogID.A, $"wrote to switch board without time outs after {tryingToRecoverAfterTimeoutWatch.Elapsed}, resuming normal action frequency - {board.ToShortDescription()}");
}
}
catch (TimeoutException)
{
if (!tryingToRecoverAfterTimeoutWatch.IsRunning)
{
// we only notify of the situation once while this is a different way a disconnect might look like, we have
CALog.LogInfoAndConsoleLn(LogID.A, $"timed out writing to switchboard, reducing action frequency until reconnect - {board.ToShortDescription()}");
tryingToRecoverAfterTimeoutWatch.Restart();
}
// forcing reduced acting frequency )
try { await Task.Delay(500, token); }
catch (TaskCanceledException) { }
}
catch (TaskCanceledException ex)
{
if (!token.IsCancellationRequested)
{
CALog.LogErrorAndConsoleLn(LogID.A, ex.ToString());
}
}
catch (Exception ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, ex.ToString());
}
}
AllOff(board, ports);
}
private static void AllOff(MCUBoard board, List <IOconfOut230Vac> ports)
{
try
{
board.SafeWriteLine("off", CancellationToken.None);
foreach (var port in ports)
{
if (port.HasOnSafeState)
{
board.SafeWriteLine($"p{port.PortNumber} on", CancellationToken.None);
}
}
}
catch (Exception ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, $"Error detected while attempting to set ports to default positions for board {board.ToShortDescription()}", ex);
}
}
private static async Task <MCUBoard> AttemptToOpenDeviceConnection(string name, CALogLevel logLevel)
{
try
{
var mcu = await MCUBoard.OpenDeviceConnection(name);
mcu.productType = mcu.productType ?? GetStringFromDmesg(mcu.PortName);
string logline = logLevel == CALogLevel.Debug ? mcu.ToDebugString(Environment.NewLine) : mcu.ToString();
CALog.LogInfoAndConsoleLn(LogID.A, logline);
return(mcu);
}
catch (UnauthorizedAccessException ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, $"Unable to open {name}, Exception: {ex.Message}" + Environment.NewLine);
}
catch (Exception ex)
{
CALog.LogErrorAndConsoleLn(LogID.A, $"Unable to open {name}, Exception: {ex}" + Environment.NewLine);
}
return(default);
private void LoopForever()
{
_running = true;
var vectorUploadDelay = IOconfFile.GetVectorUploadDelay();
var throttle = new TimeThrottle(vectorUploadDelay);
while (_running)
{
throttle.Wait();
try
{
var list = DequeueAllEntries(_queue);
if (list != null)
{
PostVectorAsync(GetSignedVectors(list), list.First().timestamp);
}
var events = DequeueAllEntries(_eventsQueue);
if (events != null)
{
foreach (var @event in events)
{
PostEventAsync(@event);
}
}
PrintBadPackagesMessage(false);
}
catch (Exception ex)
{
if (!Environment.MachineName.StartsWith("DESKTOP"))
{
CALog.LogErrorAndConsoleLn(LogID.A, "ServerUploader.LoopForever() exception: " + ex.Message, ex);
}
}
}
PrintBadPackagesMessage(true);
}
private static async Task DoPortActions(NewVectorReceivedArgs vector, MCUBoard board, IOconfOut230Vac port, SwitchboardAction[] lastActions, CancellationToken token)
{
if (token.IsCancellationRequested)
{
return;
}
try
{
var action = SwitchboardAction.FromVectorSamples(vector, port.Name);
if (action.Equals(lastActions[port.PortNumber - 1]))
{
return; // no action changes has been requested since the last action taken on the heater.
}
var onSeconds = action.GetRemainingOnSeconds(vector.GetVectorTime());
if (onSeconds <= 0)
{
await board.SafeWriteLine($"p{port.PortNumber} off", token);
}
else if (onSeconds == int.MaxValue)
{
await board.SafeWriteLine($"p{port.PortNumber} on", token);
}
else
{
await board.SafeWriteLine($"p{port.PortNumber} on {onSeconds}", token);
}
lastActions[port.PortNumber - 1] = action;
}
catch (TimeoutException)
{
// we don't want logging at this level as the caller handles this in a way that reduces the amount of noise for failures that last many vectors.
throw;
}
catch (Exception)
{
CALog.LogErrorAndConsoleLn(LogID.A, $"Failed executing port action for board {board.BoxName} port {port.Name} - {port.Name}");
throw; // this will log extra info and avoid extra board actions on this cycle
}
}
private static void OnError(string message, Exception ex) => CALog.LogErrorAndConsoleLn(LogID.A, message, ex);
public void LogError(string message) => CALog.LogErrorAndConsoleLn(LogID.A, FormatMessage(message));
