private async Task ShutdownAdapter(CalcInstance adapter)
{
adapter.State = State.ShutdownStarted;
try {
while (adapter.RunLoopRunning)
{
await Task.Delay(50);
}
var instance = adapter.Instance;
if (instance == null)
{
Log_Warn("CalcShutdownError", "ShutdownCalc: Instance is null");
}
else
{
await instance.Shutdown();
}
}
catch (Exception e) {
Exception exp = e.GetBaseException() ?? e;
Log_Warn("CalcShutdownError", "ShutdownCalc exception: " + exp.Message);
}
adapter.State = State.ShutdownCompleted;
adapter.SetInstanceNull();
}
private void StartRunLoopTask(CalcInstance adapter)
{
Task readTask = AdapterRunLoopTask(adapter);
adapter.RunLoopTask = readTask;
var ignored1 = readTask.ContinueOnMainThread(t => {
if (t.IsFaulted && !moduleShutdown && adapter.State == State.Running)
{
Exception exp = t.Exception !.GetBaseException() ?? t.Exception;
Task ignored2 = RestartAdapter(adapter, "Run loop exception: " + exp.Message + "\n" + exp.StackTrace, critical: true);
}
});
}
private void Do_Notify_NeedRestart(string reason, Calculation adapter)
{
CalcInstance ast = adapters.FirstOrDefault(a => a.CalcConfig.ID == adapter.ID);
Task ignored = RestartAdapter(ast, reason);
}
private Timestamp GetNextNormalizedTimestamp(Timestamp tCurrent, Duration cycle, Duration offset, CalcInstance adapter)
{
Timestamp tNext = Time.GetNextNormalizedTimestamp(cycle, offset);
Duration minDuration = Duration.FromMilliseconds(1);
Duration c = cycle < minDuration ? minDuration : cycle;
while (tNext <= tCurrent)
{
tNext += c;
}
Trigger trigger = adapter.triggerDurationWarning.GetTrigger(isOK: tNext == tCurrent + cycle);
if (trigger == Trigger.On)
{
Log_Warn("Cycle", $"Cycle length of {cycle} not long enough for calculation {adapter.Name}!", null, new ObjectRef[] { adapter.ID });
}
else if (trigger == Trigger.Off)
{
Log_ReturnToNormal("Cycle", $"Cycle length of {cycle} is long enough for calculation {adapter.Name}.", new ObjectRef[] { adapter.ID });
}
return(tNext);
}
private async Task <VTQs> ReadInputVars(CalcInstance adapter, List <Config.Input> inputs, List <VariableRef> inputVars, Timestamp now)
{
try {
return(await connection.ReadVariables(inputVars));
}
catch (Exception exp) {
Exception e = exp.GetBaseException() ?? exp;
if (e is ConnectivityException)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
while (!moduleShutdown && sw.ElapsedMilliseconds < 5 * 1000)
{
await Task.Delay(10);
}
sw.Stop();
if (moduleShutdown)
{
throw;
}
await RestartConnectionOrFail();
return(await ReadInputVars(adapter, inputs, inputVars, now));
}
else if (e is RequestException)
{
int N = inputs.Count;
var invalidInputVars = new List <int>();
VTQs res = new VTQs(N);
for (int i = 0; i < N; ++i)
{
VariableRef variable = inputVars[i];
try {
res.Add(await connection.ReadVariable(variable));
}
catch (Exception) {
res.Add(VTQ.Make(inputs[i].GetDefaultValue(), now, Quality.Bad));
invalidInputVars.Add(i);
}
}
var affectedObjects = new ObjectRef[] { adapter.ID };
if (invalidInputVars.Count == 1)
{
int i = invalidInputVars[0];
var input = inputs[i];
string details = input.Name + ": " + inputVars[i].ToString();
Log_ErrorDetails("Invalid_Input_Var", $"Invalid variable ref for input '{input.Name}' of calculation '{adapter.Name}'", details, affectedObjects);
}
else if (invalidInputVars.Count > 1)
{
string[] items = invalidInputVars.Select(i => inputs[i].Name + ": " + inputVars[i].ToString()).ToArray();
string details = string.Join("; ", items);
Log_ErrorDetails("Invalid_Input_Var", $"Invalid variable refs for inputs of calculation '{adapter.Name}'", details, affectedObjects);
}
return(res);
}
else
{
throw e;
}
}
}
private async Task AdapterRunLoopTask(CalcInstance adapter)
{
adapter.State = State.Running;
Duration cycle = adapter.ScaledCycle();
Duration offset = adapter.ScaledOffset();
Timestamp t = Time.GetNextNormalizedTimestamp(cycle, offset);
string moduleID = base.moduleID;
await adapter.WaitUntil(t);
var inputs = new List <Config.Input>();
var inputVars = new List <VariableRef>();
var notifier = this.notifier !;
//var listVarValueTimer = new List<VariableValue>(1);
//var sw = System.Diagnostics.Stopwatch.StartNew();
while (adapter.State == State.Running)
{
//sw.Restart();
inputs.Clear();
inputVars.Clear();
foreach (Config.Input inp in adapter.CalcConfig.Inputs)
{
if (inp.Variable.HasValue)
{
inputs.Add(inp);
inputVars.Add(inp.Variable.Value);
}
}
// Config.Input[] inputs = adapter.CalcConfig.Inputs.Where(inp => inp.Variable.HasValue).ToArray();
// VariableRef[] inputVars = inputs.Select(inp => inp.Variable.Value).ToArray();
VTQs values = await ReadInputVars(adapter, inputs, inputVars, t);
// sw.Stop();
// double dd = sw.ElapsedTicks;
//if (dd > 100) {
// Console.Error.WriteLine(dd / TimeSpan.TicksPerMillisecond + " ms");
//}
adapter.UpdateInputValues(inputVars, values);
InputValue[] inputValues = adapter.CurrentInputValues(t);
List <VariableValue> inValues = inputValues.Select(v => VariableValue.Make(adapter.GetInputVarRef(v.InputID), v.Value.WithTime(t))).ToList();
notifier.Notify_VariableValuesChanged(inValues);
var instance = adapter.Instance;
if (instance == null || adapter.State != State.Running)
{
break;
}
StepResult result = await instance.Step(t, inputValues);
OutputValue[] outValues = result.Output ?? new OutputValue[0];
StateValue[] stateValues = result.State ?? new StateValue[0];
// Console.WriteLine($"{Timestamp.Now}: out: " + StdJson.ObjectToString(outValues));
var listVarValues = new List <VariableValue>(outValues.Length + stateValues.Length);
foreach (OutputValue v in outValues)
{
var vv = VariableValue.Make(adapter.GetOutputVarRef(v.OutputID), v.Value);
listVarValues.Add(vv);
}
foreach (StateValue v in stateValues)
{
var vv = VariableValue.Make(adapter.GetStateVarRef(v.StateID), VTQ.Make(v.Value, t, Quality.Good));
listVarValues.Add(vv);
}
List <VariableValue> outputDest = new List <VariableValue>();
foreach (Config.Output ot in adapter.CalcConfig.Outputs)
{
if (ot.Variable.HasValue)
{
int idx = outValues.FindIndex(o => o.OutputID == ot.ID);
if (idx > -1)
{
outputDest.Add(VariableValue.Make(ot.Variable.Value, outValues[idx].Value));
}
}
}
notifier.Notify_VariableValuesChanged(listVarValues);
await WriteOutputVars(outputDest);
adapter.SetLastOutputValues(outValues);
adapter.SetLastStateValues(stateValues);
//sw.Stop();
//var vvv1 = VariableValue.Make(adapter.GetLastRunDurationVarRef(), VTQ.Make(sw.ElapsedMilliseconds, t, Quality.Good));
//listVarValueTimer.Clear();
//listVarValueTimer.Add(vvv1);
//notifier.Notify_VariableValuesChanged(listVarValueTimer);
t = GetNextNormalizedTimestamp(t, cycle, offset, adapter);
await adapter.WaitUntil(t);
}
}
private async Task RestartAdapter(CalcInstance adapter, string reason, bool critical = true, int tryCounter = 0)
{
if (adapter.IsRestarting && tryCounter == 0)
{
return;
}
adapter.IsRestarting = true;
if (critical)
{
if (tryCounter == 0)
{
Log_Warn("CalcRestart", $"Restarting calculation {adapter.Name}. Reason: {reason}");
}
else
{
Log_Warn("CalcRestart", $"Restarting calculation {adapter.Name} (retry {tryCounter}). Reason: {reason}");
}
}
else
{
Log_Info("CalcRestart", $"Restarting calculation {adapter.Name}. Reason: {reason}");
}
const int TimeoutSeconds = 10;
try {
Task shutdown = ShutdownAdapter(adapter);
Task t = await Task.WhenAny(shutdown, Task.Delay(TimeSpan.FromSeconds(TimeoutSeconds)));
if (t != shutdown)
{
Log_Warn("CalcShutdownTimeout", $"Shutdown request for calculation {adapter.Name} failed to complete within {TimeoutSeconds} seconds.");
// go ahead and hope for the best...
}
adapter.CreateInstance(mapAdapterTypes);
await InitAdapter(adapter);
if (adapter.State == State.InitComplete)
{
StartRunLoopTask(adapter);
}
adapter.IsRestarting = false;
}
catch (Exception exception) {
Exception exp = exception.GetBaseException();
string errMsg = $"Restart of calculation {adapter.Name} failed: {exp.Message}";
if (critical)
{
Log_Error("CalcRestartError", errMsg);
// Thread.Sleep(500);
// Environment.Exit(1); // will result in restart of entire module by Mediator
int delayMS = Math.Min(10 * 1000, (tryCounter + 1) * 1000);
await Task.Delay(delayMS);
Task _ = RestartAdapter(adapter, exp.Message, critical, tryCounter + 1);
}
else
{
Log_Info("CalcRestartFailed", errMsg);
adapter.IsRestarting = false;
throw new Exception(errMsg);
}
}
}
private async Task InitAdapter(CalcInstance adapter)
{
if (adapter.Instance == null)
{
throw new Exception("InitAdapter: instance is null");
}
Calculation info = adapter.CalcConfig.ToCalculation();
try {
var initParams = new InitParameter()
{
Calculation = info,
LastOutput = adapter.LastOutputValues,
LastState = adapter.LastStateValues,
ConfigFolder = Path.GetDirectoryName(base.modelFileName) ?? "",
DataFolder = initInfo.DataFolder,
ModuleConfig = moduleConfig.ToNamedValues()
};
adapter.State = State.InitStarted;
InitResult res = await adapter.Instance.Initialize(initParams, new Wrapper(this, info));
if (adapter.State == State.InitStarted)
{
Config.Input[] newInputs = res.Inputs.Select(ip => MakeInput(ip, adapter.CalcConfig)).ToArray();
Config.Output[] newOutputs = res.Outputs.Select(ip => MakeOutput(ip, adapter.CalcConfig)).ToArray();
Config.State[] newStates = res.States.Select(MakeState).ToArray();
bool inputsChanged = !StdJson.ObjectsDeepEqual(adapter.CalcConfig.Inputs, newInputs);
bool outputsChanged = !StdJson.ObjectsDeepEqual(adapter.CalcConfig.Outputs, newOutputs);
bool statesChanged = !StdJson.ObjectsDeepEqual(adapter.CalcConfig.States, newStates);
var changedMembers = new List <MemberValue>(2);
if (inputsChanged)
{
changedMembers.Add(MemberValue.Make(moduleID, adapter.CalcConfig.ID, "Inputs", DataValue.FromObject(newInputs)));
}
if (outputsChanged)
{
changedMembers.Add(MemberValue.Make(moduleID, adapter.CalcConfig.ID, "Outputs", DataValue.FromObject(newOutputs)));
}
if (statesChanged)
{
changedMembers.Add(MemberValue.Make(moduleID, adapter.CalcConfig.ID, "States", DataValue.FromObject(newStates)));
}
if (changedMembers.Count > 0)
{
await UpdateConfig(GetModuleOrigin(),
updateOrDeleteObjects : new ObjectValue[0],
updateOrDeleteMembers : changedMembers.ToArray(),
addArrayElements : new AddArrayElement[0]);
}
adapter.State = State.InitComplete;
}
}
catch (Exception e) {
Exception exp = e.GetBaseException() ?? e;
adapter.State = State.InitError;
adapter.LastError = exp.Message;
throw new Exception($"Initialize of calculation {info.Name} failed: " + exp.Message, exp);
}
}
