/// <summary> /// Test code. Currently configured for 703810 using 2A100 versus LC0. /// </summary> public static void Test() { string ETHERAL_EXE = @"\\synology\dev\chess\engines\Ethereal12.75-x64-popcnt-avx2.exe"; string SF11_EXE = @"\\synology\dev\chess\engines\stockfish_11_x64_bmi2.exe"; string SF12_EXE = @"\\synology\dev\chess\engines\stockfish_20090216_x64_avx2.exe"; GameEngineUCISpec specEthereal = new GameEngineUCISpec("Ethereal12", ETHERAL_EXE); GameEngineUCISpec specSF = new GameEngineUCISpec("SF12", SF12_EXE); GameEngineUCISpec specLC0 = new GameEngineUCISpec("LC0", "lc0.exe"); // 66511 //NNEvaluatorDef def1 = NNEvaluatorDefFactory.SingleNet("j92-280", NNEvaluatorType.LC0Dll,1); // NNEvaluatorDef def0 = NNEvaluatorDefFactory.FromSpecification("LC0:j92-280", "GPU:1:POOLED");// POOLED");//:POOLED"); // NNEvaluatorDef def1 = NNEvaluatorDefFactory.FromSpecification("LC0:66666", "GPU:1:POOLED");// POOLED");//:POOLED"); NNEvaluatorDef evalDef1 = NNEvaluatorDefFactory.FromSpecification("LC0:703810", "GPU:0,1"); // POOLED");//:POOLED"); NNEvaluatorDef evalDef2 = NNEvaluatorDefFactory.FromSpecification("LC0:703810", "GPU:0,1"); // POOLED");//:POOLED"); // sv5300 j104.0-10000 // def1.MakePersistent(); //def1.PersistentID = "PERSIST"; // NNEvaluatorDef def2 = NNEvaluatorDefFactory.FromSpecification("LC0:66581", "GPU:3");//:POOLED"); // SearchLimit slLC0 = SearchLimit.NodesPerMove(10_000); // SearchLimit slEthereal = slLC0 * 875; // SearchLimit slSF = slLC0 * 875; //specEthereal = specSF; string[] extraUCI = null;// new string[] {"setoption name Contempt value 5000" }; const int NUM_THREADS = 32; const int HASH_SIZE_MB = 2048; string TB_PATH = CeresUserSettingsManager.Settings.DirTablebases; SearchLimit limit = SearchLimit.NodesPerMove(1_000); //limit = SearchLimit.SecondsForAllMoves(60); //limit = SearchLimit.SecondsForAllMoves(60, 1f); limit = SearchLimit.SecondsForAllMoves(120, 0.5f); GameEngineDefCeres engineDefCeres1 = new GameEngineDefCeres("Ceres1", evalDef1, new ParamsSearch(), null, new ParamsSelect(), null); GameEngineDefCeres engineDefCeres2 = new GameEngineDefCeres("Ceres2", evalDef2, new ParamsSearch(), null, new ParamsSelect(), null); bool forceDisableSmartPruning = limit.IsNodesLimit; if (forceDisableSmartPruning) { engineDefCeres1.SearchParams.FutilityPruningStopSearchEnabled = false; engineDefCeres2.SearchParams.FutilityPruningStopSearchEnabled = false; } GameEngineDef engineDefEthereal = new GameEngineDefUCI("Etheral", new GameEngineUCISpec("Etheral", ETHERAL_EXE, NUM_THREADS, HASH_SIZE_MB, TB_PATH, uciSetOptionCommands: extraUCI)); GameEngineDef engineDefStockfish11 = new GameEngineDefUCI("SF11", new GameEngineUCISpec("SF11", SF11_EXE, NUM_THREADS, HASH_SIZE_MB, TB_PATH, uciSetOptionCommands: extraUCI)); GameEngineDef engineDefStockfish12 = new GameEngineDefUCI("SF12", new GameEngineUCISpec("SF12", SF12_EXE, NUM_THREADS, HASH_SIZE_MB, TB_PATH, uciSetOptionCommands: extraUCI)); //GameEngineDef engineDefCeresUCI = new GameEngineDefUCI("CeresUCI", new GameEngineUCISpec("CeresUCI", @"c:\dev\ceres\artifacts\release\net5.0\ceres.exe")); GameEngineDef engineDefCeresUCI = new GameEngineDefCeresUCI("CeresUCI", evalDef1, overrideEXE: @"c:\dev\ceres\artifacts\release\net5.0\ceres.exe"); GameEngineDefLC0 engineDefLC1 = new GameEngineDefLC0("LC0_0", evalDef1, forceDisableSmartPruning, null, null); GameEngineDefLC0 engineDefLC2 = new GameEngineDefLC0("LC0_1", evalDef2, forceDisableSmartPruning, null, null); EnginePlayerDef playerCeres1UCI = new EnginePlayerDef(engineDefCeresUCI, limit); EnginePlayerDef playerCeres2UCI = new EnginePlayerDef(engineDefCeresUCI, limit); EnginePlayerDef playerCeres1 = new EnginePlayerDef(engineDefCeres1, limit); EnginePlayerDef playerCeres2 = new EnginePlayerDef(engineDefCeres2, limit); EnginePlayerDef playerEthereal = new EnginePlayerDef(engineDefEthereal, limit); EnginePlayerDef playerStockfish11 = new EnginePlayerDef(engineDefStockfish11, limit); EnginePlayerDef playerStockfish12 = new EnginePlayerDef(engineDefStockfish12, limit); EnginePlayerDef playerLC0 = new EnginePlayerDef(engineDefLC1, limit); EnginePlayerDef playerLC1 = new EnginePlayerDef(engineDefLC2, limit); // def.SearchLimitEngine1 = def.SearchLimitEngine2 = SearchLimit.SecondsForAllMoves(15, 0.25f); // def.SearchLimitEngine2 = def.SearchLimitEngine2 = SearchLimit.SecondsForAllMoves(15, 0.25f); //(playerCeres1.EngineDef as GameEngineDefCeres).SearchParams.DrawByRepetitionLookbackPlies = 40; if (false) { // =============================================================================== SuiteTestDef suiteDef = new SuiteTestDef("Suite", @"\\synology\dev\chess\data\epd\endgame2.epd", playerCeres1, playerCeres2, null); // suiteDef.MaxNumPositions = 100; SuiteTestRunner suiteRunner = new SuiteTestRunner(suiteDef); suiteRunner.Run(1, true, false); return; // =============================================================================== } // engineDefCeres2.SearchParams.TwofoldDrawEnabled = false; //engineDefCeres1.SearchParams.TreeReuseEnabled = false; //engineDefCeres2.SearchParams.TreeReuseEnabled = false; //engineDefCeres1.SearchParams.FutilityPruningStopSearchEnabled= false; //engineDefCeres2.SearchParams.FutilityPruningStopSearchEnabled= false; //engineDefLC0.SearchParamsEmulate.FutilityPruningStopSearchEnabled= false; //TournamentDef def = new TournamentDef("TOURN", playerCeres1, playerLC0); TournamentDef def = new TournamentDef("TOURN", playerCeres1, playerLC0); def.NumGamePairs = 50; // def.ShowGameMoves = false; // def.OpeningsFileName = @"HERT_2017\Hert500.pgn"; // def.StartingFEN = "1q6/2n4k/1r1p1pp1/RP1P2p1/2Q1P1P1/2N4P/3K4/8 b - - 8 71"; // def.OpeningsFileName = @"\\synology\dev\chess\data\openings\Drawkiller_500pos_reordered.pgn";// def.OpeningsFileName = "TCEC19_NoomenSelect.pgn"; // def.OpeningsFileName = "TCEC1819.pgn"; // def.AdjudicationThresholdCentipawns = 500; // def.AdjudicationThresholdNumMoves = 3; const int CONCURRENCY = 1;// 15; TournamentManager runner = new TournamentManager(def, CONCURRENCY); TournamentResultStats results; //UCIEngineProcess.VERBOSE = true; TimingStats stats = new TimingStats(); using (new TimingBlock(stats, TimingBlock.LoggingType.None)) { results = runner.RunTournament(); } Console.WriteLine(); Console.WriteLine($"Tournament completed in {stats.ElapsedTimeSecs,8:F2} seconds."); Console.WriteLine(results.GameOutcomesString); }
public SuiteTestResult Run(int numConcurrentSuiteThreads = 1, bool outputDetail = true, bool saveCacheWhenDone = true) { // Tree reuse is no help, indicate that we won't need it Def.Engine1Def.SearchParams.TreeReuseEnabled = false; if (Def.Engine2Def != null) { Def.Engine2Def.SearchParams.TreeReuseEnabled = false; } // Disable dump for now, the execution parameters are modified // for the warmup which is confusing because different parameters // will be chosen for the actual search. //DumpParams(Def.Output, true); // Create evaluators evaluatorSet1 = new NNEvaluatorSet(Def.Engine1Def.EvaluatorDef); if (Def.Engine2Def != null) { evaluatorSet2 = new NNEvaluatorSet(Def.Engine2Def.EvaluatorDef); } numConcurrentSuiteThreads = numConcurrentSuiteThreads; int timerFiredCount = 0; // TODO: add path automatically List <EPDEntry> epds = EPDEntry.EPDEntriesInEPDFile(Def.EPDFileName, int.MaxValue); if (Def.MaxNumPositions == 0) { Def.MaxNumPositions = epds.Count; } Def.Output.WriteLine(); Def.Output.WriteLine("C1 = " + Def.Engine1Def.EvaluatorDef); if (Def.RunCeres2Engine) { Def.Output.WriteLine("C2 = " + Def.Engine2Def.EvaluatorDef); } if (Def.ExternalEngineDef != null) { Def.Output.WriteLine("EX = " + Def.ExternalEngineDef.EngineDef); } #if NOT // To make up for the fact that LZ0 "cheats" by sometimes running over specified number of nodes // (she seems to always fill the batch even if reached limit), add half a batch extra for Ceres as compensation if (searchLimitCeres1.Type == SearchLimit.LimitType.NodesPerMove) { searchLimitCeres1 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch1.BATCH_SIZE_PRIMARY / 2); searchLimitCeres2 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch2.BATCH_SIZE_PRIMARY / 2); } #endif //Def.Output.WriteLine($"MAX_CERES_GAME_THREADS {numConcurrentCeresGames} MAX_LEELA_GAME_THREADS {MAX_LEELA_GAME_THREADS}"); // Turn of position reuse if evaluators produce different results if (Def.RunCeres2Engine && !Def.Engine1Def.EvaluatorDef.NetEvaluationsIdentical(Def.Engine2Def.EvaluatorDef)) { Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false; Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false; } if (Def.RunCeres2Engine && (Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree || Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree)) { Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine("\r\nWARNING: REUSE_POSITION_EVALUATIONS_FROM_OTHER_TREE is turned on for one or both evaluators\r\n" + "(alternating between the two evaluators). This may cause slight differences in search behavior and speed.\r\n"); Console.ForegroundColor = ConsoleColor.White; } Def.Output.WriteLine(); if (Def.MaxNumPositions > epds.Count) { Def.MaxNumPositions = epds.Count; } epds = epds.GetRange(Def.FirstTestPosition, Def.MaxNumPositions); int numExternalGameProcesses = 1; numConcurrentSuiteThreads = Math.Min(Def.MaxNumPositions, numConcurrentSuiteThreads); if (numConcurrentSuiteThreads > 1) { bool evaluator1NonPooled = Def.Engine1Def?.EvaluatorDef != null && Def.Engine1Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled; bool evaluator2NonPooled = Def.Engine2Def?.EvaluatorDef != null && Def.Engine2Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled; if (evaluator1NonPooled || evaluator2NonPooled) { throw new Exception("Must use POOLED neural network evaluator when running suites with parallelism");; } if (Def.ExternalEngineDef != null) { // For safety (to not overflow main or GPU memory) we limit number of LC0 processes. const int MAX_LC0_PROCESSES = 4; numExternalGameProcesses = Math.Min(MAX_LC0_PROCESSES, numConcurrentSuiteThreads); } } bool leelaVerboseMovesStats = true;//xxx Def.NumTestPos == 1; Func <object> makeExternalEngine = null; if (Def.ExternalEngineDef != null) { if (Def.ExternalEngineDef.EngineDef is GameEngineDefLC0) { GameEngineDefLC0 lc0EngineDef = Def.ExternalEngineDef.EngineDef as GameEngineDefLC0; bool forceDisableSmartPruning = lc0EngineDef.ForceDisableSmartPruning; makeExternalEngine = () => { LC0Engine engine = LC0EngineConfigured.GetLC0Engine(null, null, Def.Engine1Def.EvaluatorDef, NNWeightsFiles.LookupNetworkFile(Def.Engine1Def.EvaluatorDef.Nets[0].Net.NetworkID), true, false, leelaVerboseMovesStats, forceDisableSmartPruning, lc0EngineDef.OverrideEXE, false, lc0EngineDef.ExtraCommandLineArgs); // WARMUP engine.AnalyzePositionFromFEN(Position.StartPosition.FEN, SearchLimit.NodesPerMove(1)); return(engine); }; } else { //bool resetMovesBetweenMoves = !Def.Engine2Def.SearchParams.TreeReuseEnabled; //bool enableTranpsositions = Def.Engine2Def.SearchParams.Execution.TranspositionMode != TranspositionMode.None; //bool enableTablebases = Def.Engine2Def.SearchParams.EnableTablebases; makeExternalEngine = () => Def.ExternalEngineDef.EngineDef.CreateEngine(); } } // Don't create too many non_Ceres threads since each one will consume seaprate GPU memory or threads int maxLeelaThreads = Math.Min(numExternalGameProcesses, numConcurrentSuiteThreads); ObjectPool <object> externalEnginePool = new ObjectPool <object>(makeExternalEngine, maxLeelaThreads); using (new TimingBlock("EPDS")) { Parallel.For(0, epds.Count, new ParallelOptions() { MaxDegreeOfParallelism = numConcurrentSuiteThreads }, delegate(int gameNum) { try { EPDEntry epd = epds[gameNum]; // Skip positions which are already draws if (epd.Position.CheckDrawBasedOnMaterial == Position.PositionDrawStatus.DrawByInsufficientMaterial) { return; } // TODO: also do this for checkmate? ProcessEPD(gameNum, epds[gameNum], outputDetail, externalEnginePool); } catch (Exception exc) { Def.Output.WriteLine("Error in ProcessEPD " + exc); throw exc; } }); } Def.Output.WriteLine(); Def.Output.WriteLine(); if (Def.ExternalEngineDef != null) { Def.Output.WriteLine($"Total {Def.ExternalEngineDef.ID} Time {totalTimeOther,6:F2}"); } Def.Output.WriteLine($"Total C1 Time {totalTimeCeres1,6:F2}"); if (Def.CeresEngine2Def != null) { Def.Output.WriteLine($"Total C2 Time {totalTimeCeres2,6:F2}"); } Def.Output.WriteLine(); if (Def.ExternalEngineDef != null) { Def.Output.WriteLine($"Avg {Def.ExternalEngineDef.ID} pos/sec {totalNodesOther / totalTimeOther,8:F2}"); } Def.Output.WriteLine($"Avg Ceres pos/sec {totalNodes1 / totalTimeCeres1,8:F2}"); if (Def.CeresEngine2Def != null) { Def.Output.WriteLine($"Avg Ceres2 pos/sec {totalNodes2 / totalTimeCeres2,8:F2}"); } Def.Output.WriteLine(); Def.Output.WriteLine(); EngineCeres1.Dispose(); EngineCeres2?.Dispose(); EngineExternal?.Dispose(); externalEnginePool.Shutdown(engineObj => (engineObj as LC0Engine).Dispose()); evaluatorSet1.Dispose(); evaluatorSet2?.Dispose(); return(new SuiteTestResult(Def) { AvgScore1 = (float)accCeres1 / numSearches, AvgScore2 = (float)accCeres2 / numSearches, AvgWScore1 = (float)accWCeres1 / numSearches, AvgWScore2 = (float)accWCeres2 / numSearches, AvgScoreLC0 = (float)avgOther / numSearches, TotalRuntimeLC0 = totalTimeOther, TotalRuntime1 = totalTimeCeres1, TotalRuntime2 = totalTimeCeres2, TotalNodesLC0 = totalNodesOther, TotalNodes1 = totalNodes1, TotalNodes2 = totalNodes2 }); }