/// <summary> /// Constructor. /// </summary> /// <param name="def"></param> public SuiteTestRunner(SuiteTestDef def) { Def = def; // Create and warmup both engines (in parallel) Parallel.Invoke(() => { EngineCeres1 = Def.Engine1Def.CreateEngine() as GameEngineCeresInProcess; EngineCeres1.Warmup(); }, () => { EngineCeres2 = Def.Engine2Def?.CreateEngine() as GameEngineCeresInProcess; EngineCeres2?.Warmup(); }, () => { EngineExternal = Def.Engine2Def?.CreateEngine(); EngineExternal?.Warmup(); }); }
public SuiteTestResult Run(int numConcurrentSuiteThreads = 1, bool outputDetail = true, bool saveCacheWhenDone = true) { // Tree reuse is no help, indicate that we won't need it Def.Engine1Def.SearchParams.TreeReuseEnabled = false; if (Def.Engine2Def != null) { Def.Engine2Def.SearchParams.TreeReuseEnabled = false; } // Disable dump for now, the execution parameters are modified // for the warmup which is confusing because different parameters // will be chosen for the actual search. //DumpParams(Def.Output, true); // Create evaluators evaluatorSet1 = new NNEvaluatorSet(Def.Engine1Def.EvaluatorDef); if (Def.Engine2Def != null) { evaluatorSet2 = new NNEvaluatorSet(Def.Engine2Def.EvaluatorDef); } numConcurrentSuiteThreads = numConcurrentSuiteThreads; int timerFiredCount = 0; // TODO: add path automatically List <EPDEntry> epds = EPDEntry.EPDEntriesInEPDFile(Def.EPDFileName, int.MaxValue); if (Def.MaxNumPositions == 0) { Def.MaxNumPositions = epds.Count; } Def.Output.WriteLine(); Def.Output.WriteLine("C1 = " + Def.Engine1Def.EvaluatorDef); if (Def.RunCeres2Engine) { Def.Output.WriteLine("C2 = " + Def.Engine2Def.EvaluatorDef); } if (Def.ExternalEngineDef != null) { Def.Output.WriteLine("EX = " + Def.ExternalEngineDef.EngineDef); } #if NOT // To make up for the fact that LZ0 "cheats" by sometimes running over specified number of nodes // (she seems to always fill the batch even if reached limit), add half a batch extra for Ceres as compensation if (searchLimitCeres1.Type == SearchLimit.LimitType.NodesPerMove) { searchLimitCeres1 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch1.BATCH_SIZE_PRIMARY / 2); searchLimitCeres2 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch2.BATCH_SIZE_PRIMARY / 2); } #endif //Def.Output.WriteLine($"MAX_CERES_GAME_THREADS {numConcurrentCeresGames} MAX_LEELA_GAME_THREADS {MAX_LEELA_GAME_THREADS}"); // Turn of position reuse if evaluators produce different results if (Def.RunCeres2Engine && !Def.Engine1Def.EvaluatorDef.NetEvaluationsIdentical(Def.Engine2Def.EvaluatorDef)) { Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false; Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false; } if (Def.RunCeres2Engine && (Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree || Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree)) { Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine("\r\nWARNING: REUSE_POSITION_EVALUATIONS_FROM_OTHER_TREE is turned on for one or both evaluators\r\n" + "(alternating between the two evaluators). This may cause slight differences in search behavior and speed.\r\n"); Console.ForegroundColor = ConsoleColor.White; } Def.Output.WriteLine(); if (Def.MaxNumPositions > epds.Count) { Def.MaxNumPositions = epds.Count; } epds = epds.GetRange(Def.FirstTestPosition, Def.MaxNumPositions); int numExternalGameProcesses = 1; numConcurrentSuiteThreads = Math.Min(Def.MaxNumPositions, numConcurrentSuiteThreads); if (numConcurrentSuiteThreads > 1) { bool evaluator1NonPooled = Def.Engine1Def.EvaluatorDef != null && Def.Engine1Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled; bool evaluator2NonPooled = Def.Engine2Def.EvaluatorDef != null && Def.Engine2Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled; if (evaluator1NonPooled || evaluator2NonPooled) { throw new Exception("Must use POOLED neural network evaluator when running suites with parallelism");; } if (Def.ExternalEngineDef != null) { // For safety (to not overflow main or GPU memory) we limit number of LC0 processes. const int MAX_LC0_PROCESSES = 4; numExternalGameProcesses = Math.Min(MAX_LC0_PROCESSES, numConcurrentSuiteThreads); } } bool leelaVerboseMovesStats = true;//xxx Def.NumTestPos == 1; Func <object> makeExternalEngine = null; if (Def.ExternalEngineDef != null) { if (Def.ExternalEngineDef.EngineDef is GameEngineDefLC0) { bool forceDisableSmartPruning = (Def.ExternalEngineDef.EngineDef as GameEngineDefLC0).ForceDisableSmartPruning; makeExternalEngine = () => { LC0Engine engine = LC0EngineConfigured.GetLC0Engine(null, null, Def.Engine1Def.EvaluatorDef, NNWeightsFiles.LookupNetworkFile(Def.Engine1Def.EvaluatorDef.Nets[0].Net.NetworkID), true, false, leelaVerboseMovesStats, forceDisableSmartPruning); // WARMUP engine.AnalyzePositionFromFEN(Position.StartPosition.FEN, null, SearchLimit.NodesPerMove(1)); return(engine); }; } else { bool resetMovesBetweenMoves = !Def.Engine2Def.SearchParams.TreeReuseEnabled; bool enableTranpsositions = Def.Engine2Def.SearchParams.Execution.TranspositionMode != TranspositionMode.None; bool enableTablebases = Def.Engine2Def.SearchParams.EnableTablebases; makeExternalEngine = () => Def.ExternalEngineDef.EngineDef.CreateEngine(); } } // Don't create too many non_Ceres threads since each one will consume seaprate GPU memory or threads int maxLeelaThreads = Math.Min(numExternalGameProcesses, numConcurrentSuiteThreads); ObjectPool <object> externalEnginePool = new ObjectPool <object>(makeExternalEngine, maxLeelaThreads); using (new TimingBlock("EPDS")) { Parallel.For(0, epds.Count, new ParallelOptions() { MaxDegreeOfParallelism = numConcurrentSuiteThreads }, delegate(int gameNum) { try { EPDEntry epd = epds[gameNum]; // Skip positions which are already draws if (epd.Position.CheckDrawBasedOnMaterial == Position.PositionDrawStatus.DrawByInsufficientMaterial) { return; } // TODO: also do this for checkmate? ProcessEPD(gameNum, epds[gameNum], outputDetail, externalEnginePool); } catch (Exception exc) { Def.Output.WriteLine("Error in ProcessEPD " + exc); throw exc; } }); } Def.Output.WriteLine(); Def.Output.WriteLine(); if (Def.ExternalEngineDef != null) { Def.Output.WriteLine($"Total {Def.ExternalEngineDef.ID} Time {totalTimeOther,6:F2}"); } Def.Output.WriteLine($"Total C1 Time {totalTimeCeres1,6:F2}"); if (Def.CeresEngine2Def != null) { Def.Output.WriteLine($"Total C2 Time {totalTimeCeres2,6:F2}"); } Def.Output.WriteLine(); if (Def.ExternalEngineDef != null) { Def.Output.WriteLine($"Avg {Def.ExternalEngineDef.ID} pos/sec {totalNodesOther / totalTimeOther,8:F2}"); } Def.Output.WriteLine($"Avg Ceres pos/sec {totalNodes1 / totalTimeCeres1,8:F2}"); if (Def.CeresEngine2Def != null) { Def.Output.WriteLine($"Avg Ceres2 pos/sec {totalNodes2 / totalTimeCeres2,8:F2}"); } Def.Output.WriteLine(); Def.Output.WriteLine(); EngineCeres1.Dispose(); EngineCeres2?.Dispose(); EngineExternal?.Dispose(); externalEnginePool.Shutdown(engineObj => (engineObj as LC0Engine).Dispose()); evaluatorSet1.Dispose(); evaluatorSet2?.Dispose(); return(new SuiteTestResult(Def) { AvgScore1 = (float)accCeres1 / numSearches, AvgScore2 = (float)accCeres2 / numSearches, AvgWScore1 = (float)accWCeres1 / numSearches, AvgWScore2 = (float)accWCeres2 / numSearches, AvgScoreLC0 = (float)avgOther / numSearches, TotalRuntimeLC0 = totalTimeOther, TotalRuntime1 = totalTimeCeres1, TotalRuntime2 = totalTimeCeres2, TotalNodesLC0 = totalNodesOther, TotalNodes1 = totalNodes1, TotalNodes2 = totalNodes2 }); }