private async Task MinerSpeedsLoop(CancellationTokenSource ct)
{
Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"STARTING");
try
{
var workers = string.Join(",", _miningPairs.Select((_, i) => $@"""{i}"""));
var workersReset = @"{""id"":1,""method"":""workers.reset"",""params"":[__WORKERS__]}".Replace("__WORKERS__", workers);
Func <bool> isActive = () => !ct.Token.IsCancellationRequested;
while (isActive())
{
try
{
_ = await ApiDataHelpers.GetApiDataAsync(_apiPort, workersReset + "\r\n", _logGroup);
if (isActive())
{
await ExcavatorTaskHelpers.TryDelay(TimeSpan.FromSeconds(30), ct.Token);
}
// get speeds
var ad = await GetMinerStatsDataAsyncPrivate();
lock (_lock2)
{
_lastApiData = ad;
}
// speed print and reset
_ = await ApiDataHelpers.GetApiDataAsync(_apiPort, @"{""id"":1,""method"":""worker.print.efficiencies"",""params"":[]}" + "\r\n", _logGroup);
}
catch (TaskCanceledException e)
{
break;
}
catch (Exception e)
{
}
}
Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"EXIT WHILE");
}
catch (TaskCanceledException e)
{
}
catch (Exception ex)
{
Logger.Error("EXCAVATOR-API_LOOP", $"error {ex}");
}
finally
{
ct.Dispose();
}
}
public override async Task <BenchmarkResult> StartBenchmark(CancellationToken stop, BenchmarkPerformanceType benchmarkType = BenchmarkPerformanceType.Standard)
{
using (var tickCancelSource = new CancellationTokenSource())
{
var workers = string.Join(",", _miningPairs.Select((_, i) => $@"""{i}"""));
var workersReset = @"{""id"":1,""method"":"" workers.reset"",""params"":[__WORKERS__]}".Replace("__WORKERS__", workers);
// determine benchmark time
// settup times
var benchmarkTime = MinerBenchmarkTimeSettings.ParseBenchmarkTime(new List <int> {
20, 40, 60
}, MinerBenchmarkTimeSettings, _miningPairs, benchmarkType); // in seconds
var maxTicks = MinerBenchmarkTimeSettings.ParseBenchmarkTicks(new List <int> {
1, 3, 9
}, MinerBenchmarkTimeSettings, _miningPairs, benchmarkType);
var maxTicksEnabled = MinerBenchmarkTimeSettings.MaxTicksEnabled;
//// use demo user and disable the watchdog
var commandLine = MiningCreateCommandLine();
var(binPath, binCwd) = GetBinAndCwdPaths();
Logger.Info(_logGroup, $"Benchmarking started with command: {commandLine}");
Logger.Info(_logGroup, $"Benchmarking settings: time={benchmarkTime} ticks={maxTicks} ticksEnabled={maxTicksEnabled}");
var bp = new BenchmarkProcess(binPath, binCwd, commandLine, GetEnvironmentVariables());
// disable line readings and read speeds from API
bp.CheckData = null;
var benchmarkTimeout = TimeSpan.FromSeconds(benchmarkTime + 5);
var benchmarkWait = TimeSpan.FromMilliseconds(500);
var t = MinerToolkit.WaitBenchmarkResult(bp, benchmarkTimeout, benchmarkWait, stop, tickCancelSource.Token);
var stoppedAfterTicks = false;
var validTicks = 0;
var ticks = benchmarkTime / 10; // on each 10 seconds tick
var result = new BenchmarkResult();
var benchmarkApiData = new List <ApiData>();
for (var tick = 0; tick < ticks; tick++)
{
if (t.IsCompleted || t.IsCanceled || stop.IsCancellationRequested)
{
break;
}
_ = await ExecuteCommand(workersReset, stop);
await ExcavatorTaskHelpers.TryDelay(TimeSpan.FromSeconds(10), stop);
if (t.IsCompleted || t.IsCanceled || stop.IsCancellationRequested)
{
break;
}
// get speeds
var ad = await GetMinerStatsDataAsyncPrivate(stop);
var adTotal = ad.AlgorithmSpeedsTotal();
var isTickValid = adTotal.Count > 0 && adTotal.All(pair => pair.speed > 0);
benchmarkApiData.Add(ad);
if (isTickValid)
{
++validTicks;
}
if (maxTicksEnabled && validTicks >= maxTicks)
{
stoppedAfterTicks = true;
break;
}
}
// await benchmark task
if (stoppedAfterTicks)
{
try
{
tickCancelSource.Cancel();
}
catch
{ }
}
await t;
if (stop.IsCancellationRequested)
{
return(t.Result);
}
// calc speeds
// TODO calc std deviaton to reduce invalid benches
try
{
var nonZeroSpeeds = benchmarkApiData.Where(ad => ad.AlgorithmSpeedsTotal().Count > 0 && ad.AlgorithmSpeedsTotal().All(pair => pair.speed > 0))
.Select(ad => (ad, ad.AlgorithmSpeedsTotal().Count)).ToList();
var speedsFromTotals = new List <(AlgorithmType type, double speed)>();
if (nonZeroSpeeds.Count > 0)
{
var maxAlgoPiarsCount = nonZeroSpeeds.Select(adCount => adCount.Count).Max();
var sameCountApiDatas = nonZeroSpeeds.Where(adCount => adCount.Count == maxAlgoPiarsCount).Select(adCount => adCount.ad).ToList();
var firstPair = sameCountApiDatas.FirstOrDefault();
var speedSums = firstPair.AlgorithmSpeedsTotal().Select(pair => new KeyValuePair <AlgorithmType, double>(pair.type, 0.0)).ToDictionary(x => x.Key, x => x.Value);
// sum
foreach (var ad in sameCountApiDatas)
{
foreach (var pair in ad.AlgorithmSpeedsTotal())
{
speedSums[pair.type] += pair.speed;
}
}
// average
foreach (var algoId in speedSums.Keys.ToArray())
{
speedSums[algoId] /= sameCountApiDatas.Count;
}
result = new BenchmarkResult
{
AlgorithmTypeSpeeds = firstPair.AlgorithmSpeedsTotal().Select(pair => (pair.type, speedSums[pair.type])).ToList(),
Success = true
};
}
}
catch (Exception e)
{
Logger.Warn(_logGroup, $"benchmarking AlgorithmSpeedsTotal error {e.Message}");
}
// return API result
return(result);
}
}
private async Task MinerSpeedsLoop(CancellationTokenSource ct)
{
Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"STARTING");
try
{
var workers = string.Join(",", _miningPairs.Select((_, i) => $@"""{i}"""));
var workersReset = @"{""id"":1,""method"":""workers.reset"",""params"":[__WORKERS__]}".Replace("__WORKERS__", workers);
Func <bool> isActive = () => !ct.Token.IsCancellationRequested;
var lastSuccessfulSpeeds = DateTime.UtcNow;
while (isActive())
{
var elapsed = DateTime.UtcNow - lastSuccessfulSpeeds;
if (elapsed >= TimeSpan.FromSeconds(50))
{
Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"Restaring excavator due to speed anomaly");
//_ = await ExecuteCommand(@"{""id"":1,""method"":""quit"",""params"":[]}");
#warning if no nhm watchdog it will never restart 'nhms'
await StopMiningTask();
}
try
{
_ = await ExecuteCommand(workersReset, ct.Token);
if (isActive())
{
await ExcavatorTaskHelpers.TryDelay(TimeSpan.FromSeconds(30), ct.Token);
}
// get speeds
var ad = await GetMinerStatsDataAsyncPrivate(ct.Token);
LastApiData = ad;
if (IsSpeedOk(ad))
{
lastSuccessfulSpeeds = DateTime.UtcNow;
}
// speed print and reset
_ = await ExecuteCommand(@"{""id"":1,""method"":""worker.print.efficiencies"",""params"":[]}", ct.Token);
}
catch (TaskCanceledException)
{
break;
}
catch (Exception e)
{
Logger.Error("EXCAVATOR-MinerSpeedsLoop", $"Error {e}");
}
}
Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"EXIT WHILE");
}
catch (TaskCanceledException)
{
}
catch (Exception ex)
{
Logger.Error("EXCAVATOR-API_LOOP", $"error {ex}");
}
finally
{
ct.Dispose();
}
}
private async Task MinerSpeedsLoop(CancellationTokenSource ct)
{
Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"STARTING");
try
{
int restartCount = 0;
const int MAX_RESTART_COUNT = 5; // 5 * 30s = 2.5minutes
const double PER_GPU_ANOMALY = 200 * 1000 * 1000; // 200MH/s
const double SUM_GPU_ANOMALY = 2 * 1000 * 1000 * 1000; // 2GH/s
var workers = string.Join(",", _miningPairs.Select((_, i) => $@"""{i}"""));
var workersReset = @"{""id"":1,""method"":""workers.reset"",""params"":[__WORKERS__]}".Replace("__WORKERS__", workers);
Func <bool> isActive = () => !ct.Token.IsCancellationRequested;
while (isActive())
{
try
{
_ = await ExecuteCommand(workersReset);
if (isActive())
{
await ExcavatorTaskHelpers.TryDelay(TimeSpan.FromSeconds(30), ct.Token);
}
// get speeds
var ad = await GetMinerStatsDataAsyncPrivate();
if (ad != null && ad.AlgorithmSpeedsPerDevice != null)
{
var isPerGPUAnomaly = ad.AlgorithmSpeedsPerDevice?.Values.Any((speeds) => speeds.Any((pair) => pair.speed >= PER_GPU_ANOMALY)) ?? false;
var isSumGPUAnomaly = ad.AlgorithmSpeedsPerDevice?.Values.SelectMany((speeds) => speeds.Select((pair) => pair.speed)).Sum() >= SUM_GPU_ANOMALY;
if (isPerGPUAnomaly || isSumGPUAnomaly)
{
restartCount++;
}
else
{
restartCount = 0;
}
if (restartCount >= MAX_RESTART_COUNT)
{
Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"Restaring excavator due to speed anomaly");
await ExecuteCommand(@"{""id"":1,""method"":""quit"",""params"":[]}");
}
}
lock (_lock2)
{
_lastApiData = ad;
}
// speed print and reset
_ = await ExecuteCommand(@"{""id"":1,""method"":""worker.print.efficiencies"",""params"":[]}");
}
catch (TaskCanceledException e)
{
break;
}
catch (Exception e)
{
}
}
Logger.Info("EXCAVATOR-MinerSpeedsLoop", $"EXIT WHILE");
}
catch (TaskCanceledException e)
{
}
catch (Exception ex)
{
Logger.Error("EXCAVATOR-API_LOOP", $"error {ex}");
}
finally
{
ct.Dispose();
}
}
