private async Task ShutdownAdapter(CalcInstance adapter) { adapter.State = State.ShutdownStarted; try { while (adapter.RunLoopRunning) { await Task.Delay(50); } var instance = adapter.Instance; if (instance == null) { Log_Warn("CalcShutdownError", "ShutdownCalc: Instance is null"); } else { await instance.Shutdown(); } } catch (Exception e) { Exception exp = e.GetBaseException() ?? e; Log_Warn("CalcShutdownError", "ShutdownCalc exception: " + exp.Message); } adapter.State = State.ShutdownCompleted; adapter.SetInstanceNull(); }
private void StartRunLoopTask(CalcInstance adapter) { Task readTask = AdapterRunLoopTask(adapter); adapter.RunLoopTask = readTask; var ignored1 = readTask.ContinueOnMainThread(t => { if (t.IsFaulted && !moduleShutdown && adapter.State == State.Running) { Exception exp = t.Exception !.GetBaseException() ?? t.Exception; Task ignored2 = RestartAdapter(adapter, "Run loop exception: " + exp.Message + "\n" + exp.StackTrace, critical: true); } }); }
private void Do_Notify_NeedRestart(string reason, Calculation adapter) { CalcInstance ast = adapters.FirstOrDefault(a => a.CalcConfig.ID == adapter.ID); Task ignored = RestartAdapter(ast, reason); }
private Timestamp GetNextNormalizedTimestamp(Timestamp tCurrent, Duration cycle, Duration offset, CalcInstance adapter) { Timestamp tNext = Time.GetNextNormalizedTimestamp(cycle, offset); Duration minDuration = Duration.FromMilliseconds(1); Duration c = cycle < minDuration ? minDuration : cycle; while (tNext <= tCurrent) { tNext += c; } Trigger trigger = adapter.triggerDurationWarning.GetTrigger(isOK: tNext == tCurrent + cycle); if (trigger == Trigger.On) { Log_Warn("Cycle", $"Cycle length of {cycle} not long enough for calculation {adapter.Name}!", null, new ObjectRef[] { adapter.ID }); } else if (trigger == Trigger.Off) { Log_ReturnToNormal("Cycle", $"Cycle length of {cycle} is long enough for calculation {adapter.Name}.", new ObjectRef[] { adapter.ID }); } return(tNext); }
private async Task <VTQs> ReadInputVars(CalcInstance adapter, List <Config.Input> inputs, List <VariableRef> inputVars, Timestamp now) { try { return(await connection.ReadVariables(inputVars)); } catch (Exception exp) { Exception e = exp.GetBaseException() ?? exp; if (e is ConnectivityException) { var sw = new System.Diagnostics.Stopwatch(); sw.Start(); while (!moduleShutdown && sw.ElapsedMilliseconds < 5 * 1000) { await Task.Delay(10); } sw.Stop(); if (moduleShutdown) { throw; } await RestartConnectionOrFail(); return(await ReadInputVars(adapter, inputs, inputVars, now)); } else if (e is RequestException) { int N = inputs.Count; var invalidInputVars = new List <int>(); VTQs res = new VTQs(N); for (int i = 0; i < N; ++i) { VariableRef variable = inputVars[i]; try { res.Add(await connection.ReadVariable(variable)); } catch (Exception) { res.Add(VTQ.Make(inputs[i].GetDefaultValue(), now, Quality.Bad)); invalidInputVars.Add(i); } } var affectedObjects = new ObjectRef[] { adapter.ID }; if (invalidInputVars.Count == 1) { int i = invalidInputVars[0]; var input = inputs[i]; string details = input.Name + ": " + inputVars[i].ToString(); Log_ErrorDetails("Invalid_Input_Var", $"Invalid variable ref for input '{input.Name}' of calculation '{adapter.Name}'", details, affectedObjects); } else if (invalidInputVars.Count > 1) { string[] items = invalidInputVars.Select(i => inputs[i].Name + ": " + inputVars[i].ToString()).ToArray(); string details = string.Join("; ", items); Log_ErrorDetails("Invalid_Input_Var", $"Invalid variable refs for inputs of calculation '{adapter.Name}'", details, affectedObjects); } return(res); } else { throw e; } } }
private async Task AdapterRunLoopTask(CalcInstance adapter) { adapter.State = State.Running; Duration cycle = adapter.ScaledCycle(); Duration offset = adapter.ScaledOffset(); Timestamp t = Time.GetNextNormalizedTimestamp(cycle, offset); string moduleID = base.moduleID; await adapter.WaitUntil(t); var inputs = new List <Config.Input>(); var inputVars = new List <VariableRef>(); var notifier = this.notifier !; //var listVarValueTimer = new List<VariableValue>(1); //var sw = System.Diagnostics.Stopwatch.StartNew(); while (adapter.State == State.Running) { //sw.Restart(); inputs.Clear(); inputVars.Clear(); foreach (Config.Input inp in adapter.CalcConfig.Inputs) { if (inp.Variable.HasValue) { inputs.Add(inp); inputVars.Add(inp.Variable.Value); } } // Config.Input[] inputs = adapter.CalcConfig.Inputs.Where(inp => inp.Variable.HasValue).ToArray(); // VariableRef[] inputVars = inputs.Select(inp => inp.Variable.Value).ToArray(); VTQs values = await ReadInputVars(adapter, inputs, inputVars, t); // sw.Stop(); // double dd = sw.ElapsedTicks; //if (dd > 100) { // Console.Error.WriteLine(dd / TimeSpan.TicksPerMillisecond + " ms"); //} adapter.UpdateInputValues(inputVars, values); InputValue[] inputValues = adapter.CurrentInputValues(t); List <VariableValue> inValues = inputValues.Select(v => VariableValue.Make(adapter.GetInputVarRef(v.InputID), v.Value.WithTime(t))).ToList(); notifier.Notify_VariableValuesChanged(inValues); var instance = adapter.Instance; if (instance == null || adapter.State != State.Running) { break; } StepResult result = await instance.Step(t, inputValues); OutputValue[] outValues = result.Output ?? new OutputValue[0]; StateValue[] stateValues = result.State ?? new StateValue[0]; // Console.WriteLine($"{Timestamp.Now}: out: " + StdJson.ObjectToString(outValues)); var listVarValues = new List <VariableValue>(outValues.Length + stateValues.Length); foreach (OutputValue v in outValues) { var vv = VariableValue.Make(adapter.GetOutputVarRef(v.OutputID), v.Value); listVarValues.Add(vv); } foreach (StateValue v in stateValues) { var vv = VariableValue.Make(adapter.GetStateVarRef(v.StateID), VTQ.Make(v.Value, t, Quality.Good)); listVarValues.Add(vv); } List <VariableValue> outputDest = new List <VariableValue>(); foreach (Config.Output ot in adapter.CalcConfig.Outputs) { if (ot.Variable.HasValue) { int idx = outValues.FindIndex(o => o.OutputID == ot.ID); if (idx > -1) { outputDest.Add(VariableValue.Make(ot.Variable.Value, outValues[idx].Value)); } } } notifier.Notify_VariableValuesChanged(listVarValues); await WriteOutputVars(outputDest); adapter.SetLastOutputValues(outValues); adapter.SetLastStateValues(stateValues); //sw.Stop(); //var vvv1 = VariableValue.Make(adapter.GetLastRunDurationVarRef(), VTQ.Make(sw.ElapsedMilliseconds, t, Quality.Good)); //listVarValueTimer.Clear(); //listVarValueTimer.Add(vvv1); //notifier.Notify_VariableValuesChanged(listVarValueTimer); t = GetNextNormalizedTimestamp(t, cycle, offset, adapter); await adapter.WaitUntil(t); } }
private async Task RestartAdapter(CalcInstance adapter, string reason, bool critical = true, int tryCounter = 0) { if (adapter.IsRestarting && tryCounter == 0) { return; } adapter.IsRestarting = true; if (critical) { if (tryCounter == 0) { Log_Warn("CalcRestart", $"Restarting calculation {adapter.Name}. Reason: {reason}"); } else { Log_Warn("CalcRestart", $"Restarting calculation {adapter.Name} (retry {tryCounter}). Reason: {reason}"); } } else { Log_Info("CalcRestart", $"Restarting calculation {adapter.Name}. Reason: {reason}"); } const int TimeoutSeconds = 10; try { Task shutdown = ShutdownAdapter(adapter); Task t = await Task.WhenAny(shutdown, Task.Delay(TimeSpan.FromSeconds(TimeoutSeconds))); if (t != shutdown) { Log_Warn("CalcShutdownTimeout", $"Shutdown request for calculation {adapter.Name} failed to complete within {TimeoutSeconds} seconds."); // go ahead and hope for the best... } adapter.CreateInstance(mapAdapterTypes); await InitAdapter(adapter); if (adapter.State == State.InitComplete) { StartRunLoopTask(adapter); } adapter.IsRestarting = false; } catch (Exception exception) { Exception exp = exception.GetBaseException(); string errMsg = $"Restart of calculation {adapter.Name} failed: {exp.Message}"; if (critical) { Log_Error("CalcRestartError", errMsg); // Thread.Sleep(500); // Environment.Exit(1); // will result in restart of entire module by Mediator int delayMS = Math.Min(10 * 1000, (tryCounter + 1) * 1000); await Task.Delay(delayMS); Task _ = RestartAdapter(adapter, exp.Message, critical, tryCounter + 1); } else { Log_Info("CalcRestartFailed", errMsg); adapter.IsRestarting = false; throw new Exception(errMsg); } } }
private async Task InitAdapter(CalcInstance adapter) { if (adapter.Instance == null) { throw new Exception("InitAdapter: instance is null"); } Calculation info = adapter.CalcConfig.ToCalculation(); try { var initParams = new InitParameter() { Calculation = info, LastOutput = adapter.LastOutputValues, LastState = adapter.LastStateValues, ConfigFolder = Path.GetDirectoryName(base.modelFileName) ?? "", DataFolder = initInfo.DataFolder, ModuleConfig = moduleConfig.ToNamedValues() }; adapter.State = State.InitStarted; InitResult res = await adapter.Instance.Initialize(initParams, new Wrapper(this, info)); if (adapter.State == State.InitStarted) { Config.Input[] newInputs = res.Inputs.Select(ip => MakeInput(ip, adapter.CalcConfig)).ToArray(); Config.Output[] newOutputs = res.Outputs.Select(ip => MakeOutput(ip, adapter.CalcConfig)).ToArray(); Config.State[] newStates = res.States.Select(MakeState).ToArray(); bool inputsChanged = !StdJson.ObjectsDeepEqual(adapter.CalcConfig.Inputs, newInputs); bool outputsChanged = !StdJson.ObjectsDeepEqual(adapter.CalcConfig.Outputs, newOutputs); bool statesChanged = !StdJson.ObjectsDeepEqual(adapter.CalcConfig.States, newStates); var changedMembers = new List <MemberValue>(2); if (inputsChanged) { changedMembers.Add(MemberValue.Make(moduleID, adapter.CalcConfig.ID, "Inputs", DataValue.FromObject(newInputs))); } if (outputsChanged) { changedMembers.Add(MemberValue.Make(moduleID, adapter.CalcConfig.ID, "Outputs", DataValue.FromObject(newOutputs))); } if (statesChanged) { changedMembers.Add(MemberValue.Make(moduleID, adapter.CalcConfig.ID, "States", DataValue.FromObject(newStates))); } if (changedMembers.Count > 0) { await UpdateConfig(GetModuleOrigin(), updateOrDeleteObjects : new ObjectValue[0], updateOrDeleteMembers : changedMembers.ToArray(), addArrayElements : new AddArrayElement[0]); } adapter.State = State.InitComplete; } } catch (Exception e) { Exception exp = e.GetBaseException() ?? e; adapter.State = State.InitError; adapter.LastError = exp.Message; throw new Exception($"Initialize of calculation {info.Name} failed: " + exp.Message, exp); } }