private async Task MinerSpeedsLoop(CancellationTokenSource ct) { Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"STARTING"); try { var workers = string.Join(",", _miningPairs.Select((_, i) => $@"""{i}""")); var workersReset = @"{""id"":1,""method"":""workers.reset"",""params"":[__WORKERS__]}".Replace("__WORKERS__", workers); Func <bool> isActive = () => !ct.Token.IsCancellationRequested; while (isActive()) { try { _ = await ApiDataHelpers.GetApiDataAsync(_apiPort, workersReset + "\r\n", _logGroup); if (isActive()) { await ExcavatorTaskHelpers.TryDelay(TimeSpan.FromSeconds(30), ct.Token); } // get speeds var ad = await GetMinerStatsDataAsyncPrivate(); lock (_lock2) { _lastApiData = ad; } // speed print and reset _ = await ApiDataHelpers.GetApiDataAsync(_apiPort, @"{""id"":1,""method"":""worker.print.efficiencies"",""params"":[]}" + "\r\n", _logGroup); } catch (TaskCanceledException e) { break; } catch (Exception e) { } } Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"EXIT WHILE"); } catch (TaskCanceledException e) { } catch (Exception ex) { Logger.Error("EXCAVATOR-API_LOOP", $"error {ex}"); } finally { ct.Dispose(); } }
public override async Task <BenchmarkResult> StartBenchmark(CancellationToken stop, BenchmarkPerformanceType benchmarkType = BenchmarkPerformanceType.Standard) { using (var tickCancelSource = new CancellationTokenSource()) { var workers = string.Join(",", _miningPairs.Select((_, i) => $@"""{i}""")); var workersReset = @"{""id"":1,""method"":"" workers.reset"",""params"":[__WORKERS__]}".Replace("__WORKERS__", workers); // determine benchmark time // settup times var benchmarkTime = MinerBenchmarkTimeSettings.ParseBenchmarkTime(new List <int> { 20, 40, 60 }, MinerBenchmarkTimeSettings, _miningPairs, benchmarkType); // in seconds var maxTicks = MinerBenchmarkTimeSettings.ParseBenchmarkTicks(new List <int> { 1, 3, 9 }, MinerBenchmarkTimeSettings, _miningPairs, benchmarkType); var maxTicksEnabled = MinerBenchmarkTimeSettings.MaxTicksEnabled; //// use demo user and disable the watchdog var commandLine = MiningCreateCommandLine(); var(binPath, binCwd) = GetBinAndCwdPaths(); Logger.Info(_logGroup, $"Benchmarking started with command: {commandLine}"); Logger.Info(_logGroup, $"Benchmarking settings: time={benchmarkTime} ticks={maxTicks} ticksEnabled={maxTicksEnabled}"); var bp = new BenchmarkProcess(binPath, binCwd, commandLine, GetEnvironmentVariables()); // disable line readings and read speeds from API bp.CheckData = null; var benchmarkTimeout = TimeSpan.FromSeconds(benchmarkTime + 5); var benchmarkWait = TimeSpan.FromMilliseconds(500); var t = MinerToolkit.WaitBenchmarkResult(bp, benchmarkTimeout, benchmarkWait, stop, tickCancelSource.Token); var stoppedAfterTicks = false; var validTicks = 0; var ticks = benchmarkTime / 10; // on each 10 seconds tick var result = new BenchmarkResult(); var benchmarkApiData = new List <ApiData>(); for (var tick = 0; tick < ticks; tick++) { if (t.IsCompleted || t.IsCanceled || stop.IsCancellationRequested) { break; } _ = await ExecuteCommand(workersReset, stop); await ExcavatorTaskHelpers.TryDelay(TimeSpan.FromSeconds(10), stop); if (t.IsCompleted || t.IsCanceled || stop.IsCancellationRequested) { break; } // get speeds var ad = await GetMinerStatsDataAsyncPrivate(stop); var adTotal = ad.AlgorithmSpeedsTotal(); var isTickValid = adTotal.Count > 0 && adTotal.All(pair => pair.speed > 0); benchmarkApiData.Add(ad); if (isTickValid) { ++validTicks; } if (maxTicksEnabled && validTicks >= maxTicks) { stoppedAfterTicks = true; break; } } // await benchmark task if (stoppedAfterTicks) { try { tickCancelSource.Cancel(); } catch { } } await t; if (stop.IsCancellationRequested) { return(t.Result); } // calc speeds // TODO calc std deviaton to reduce invalid benches try { var nonZeroSpeeds = benchmarkApiData.Where(ad => ad.AlgorithmSpeedsTotal().Count > 0 && ad.AlgorithmSpeedsTotal().All(pair => pair.speed > 0)) .Select(ad => (ad, ad.AlgorithmSpeedsTotal().Count)).ToList(); var speedsFromTotals = new List <(AlgorithmType type, double speed)>(); if (nonZeroSpeeds.Count > 0) { var maxAlgoPiarsCount = nonZeroSpeeds.Select(adCount => adCount.Count).Max(); var sameCountApiDatas = nonZeroSpeeds.Where(adCount => adCount.Count == maxAlgoPiarsCount).Select(adCount => adCount.ad).ToList(); var firstPair = sameCountApiDatas.FirstOrDefault(); var speedSums = firstPair.AlgorithmSpeedsTotal().Select(pair => new KeyValuePair <AlgorithmType, double>(pair.type, 0.0)).ToDictionary(x => x.Key, x => x.Value); // sum foreach (var ad in sameCountApiDatas) { foreach (var pair in ad.AlgorithmSpeedsTotal()) { speedSums[pair.type] += pair.speed; } } // average foreach (var algoId in speedSums.Keys.ToArray()) { speedSums[algoId] /= sameCountApiDatas.Count; } result = new BenchmarkResult { AlgorithmTypeSpeeds = firstPair.AlgorithmSpeedsTotal().Select(pair => (pair.type, speedSums[pair.type])).ToList(), Success = true }; } } catch (Exception e) { Logger.Warn(_logGroup, $"benchmarking AlgorithmSpeedsTotal error {e.Message}"); } // return API result return(result); } }
private async Task MinerSpeedsLoop(CancellationTokenSource ct) { Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"STARTING"); try { var workers = string.Join(",", _miningPairs.Select((_, i) => $@"""{i}""")); var workersReset = @"{""id"":1,""method"":""workers.reset"",""params"":[__WORKERS__]}".Replace("__WORKERS__", workers); Func <bool> isActive = () => !ct.Token.IsCancellationRequested; var lastSuccessfulSpeeds = DateTime.UtcNow; while (isActive()) { var elapsed = DateTime.UtcNow - lastSuccessfulSpeeds; if (elapsed >= TimeSpan.FromSeconds(50)) { Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"Restaring excavator due to speed anomaly"); //_ = await ExecuteCommand(@"{""id"":1,""method"":""quit"",""params"":[]}"); #warning if no nhm watchdog it will never restart 'nhms' await StopMiningTask(); } try { _ = await ExecuteCommand(workersReset, ct.Token); if (isActive()) { await ExcavatorTaskHelpers.TryDelay(TimeSpan.FromSeconds(30), ct.Token); } // get speeds var ad = await GetMinerStatsDataAsyncPrivate(ct.Token); LastApiData = ad; if (IsSpeedOk(ad)) { lastSuccessfulSpeeds = DateTime.UtcNow; } // speed print and reset _ = await ExecuteCommand(@"{""id"":1,""method"":""worker.print.efficiencies"",""params"":[]}", ct.Token); } catch (TaskCanceledException) { break; } catch (Exception e) { Logger.Error("EXCAVATOR-MinerSpeedsLoop", $"Error {e}"); } } Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"EXIT WHILE"); } catch (TaskCanceledException) { } catch (Exception ex) { Logger.Error("EXCAVATOR-API_LOOP", $"error {ex}"); } finally { ct.Dispose(); } }
private async Task MinerSpeedsLoop(CancellationTokenSource ct) { Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"STARTING"); try { int restartCount = 0; const int MAX_RESTART_COUNT = 5; // 5 * 30s = 2.5minutes const double PER_GPU_ANOMALY = 200 * 1000 * 1000; // 200MH/s const double SUM_GPU_ANOMALY = 2 * 1000 * 1000 * 1000; // 2GH/s var workers = string.Join(",", _miningPairs.Select((_, i) => $@"""{i}""")); var workersReset = @"{""id"":1,""method"":""workers.reset"",""params"":[__WORKERS__]}".Replace("__WORKERS__", workers); Func <bool> isActive = () => !ct.Token.IsCancellationRequested; while (isActive()) { try { _ = await ExecuteCommand(workersReset); if (isActive()) { await ExcavatorTaskHelpers.TryDelay(TimeSpan.FromSeconds(30), ct.Token); } // get speeds var ad = await GetMinerStatsDataAsyncPrivate(); if (ad != null && ad.AlgorithmSpeedsPerDevice != null) { var isPerGPUAnomaly = ad.AlgorithmSpeedsPerDevice?.Values.Any((speeds) => speeds.Any((pair) => pair.speed >= PER_GPU_ANOMALY)) ?? false; var isSumGPUAnomaly = ad.AlgorithmSpeedsPerDevice?.Values.SelectMany((speeds) => speeds.Select((pair) => pair.speed)).Sum() >= SUM_GPU_ANOMALY; if (isPerGPUAnomaly || isSumGPUAnomaly) { restartCount++; } else { restartCount = 0; } if (restartCount >= MAX_RESTART_COUNT) { Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"Restaring excavator due to speed anomaly"); await ExecuteCommand(@"{""id"":1,""method"":""quit"",""params"":[]}"); } } lock (_lock2) { _lastApiData = ad; } // speed print and reset _ = await ExecuteCommand(@"{""id"":1,""method"":""worker.print.efficiencies"",""params"":[]}"); } catch (TaskCanceledException e) { break; } catch (Exception e) { } } Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"EXIT WHILE"); } catch (TaskCanceledException e) { } catch (Exception ex) { Logger.Error("EXCAVATOR-API_LOOP", $"error {ex}"); } finally { ct.Dispose(); } }