/// <summary>
/// Test code. Currently configured for 703810 using 2A100 versus LC0.
/// </summary>
public static void Test()
{
string ETHERAL_EXE = @"\\synology\dev\chess\engines\Ethereal12.75-x64-popcnt-avx2.exe";
string SF11_EXE = @"\\synology\dev\chess\engines\stockfish_11_x64_bmi2.exe";
string SF12_EXE = @"\\synology\dev\chess\engines\stockfish_20090216_x64_avx2.exe";
GameEngineUCISpec specEthereal = new GameEngineUCISpec("Ethereal12", ETHERAL_EXE);
GameEngineUCISpec specSF = new GameEngineUCISpec("SF12", SF12_EXE);
GameEngineUCISpec specLC0 = new GameEngineUCISpec("LC0", "lc0.exe");
// 66511
//NNEvaluatorDef def1 = NNEvaluatorDefFactory.SingleNet("j92-280", NNEvaluatorType.LC0Dll,1);
// NNEvaluatorDef def0 = NNEvaluatorDefFactory.FromSpecification("LC0:j92-280", "GPU:1:POOLED");// POOLED");//:POOLED");
// NNEvaluatorDef def1 = NNEvaluatorDefFactory.FromSpecification("LC0:66666", "GPU:1:POOLED");// POOLED");//:POOLED");
NNEvaluatorDef evalDef1 = NNEvaluatorDefFactory.FromSpecification("LC0:703810", "GPU:0,1"); // POOLED");//:POOLED");
NNEvaluatorDef evalDef2 = NNEvaluatorDefFactory.FromSpecification("LC0:703810", "GPU:0,1"); // POOLED");//:POOLED");
// sv5300 j104.0-10000
// def1.MakePersistent();
//def1.PersistentID = "PERSIST";
// NNEvaluatorDef def2 = NNEvaluatorDefFactory.FromSpecification("LC0:66581", "GPU:3");//:POOLED");
// SearchLimit slLC0 = SearchLimit.NodesPerMove(10_000);
// SearchLimit slEthereal = slLC0 * 875;
// SearchLimit slSF = slLC0 * 875;
//specEthereal = specSF;
string[] extraUCI = null;// new string[] {"setoption name Contempt value 5000" };
const int NUM_THREADS = 32;
const int HASH_SIZE_MB = 2048;
string TB_PATH = CeresUserSettingsManager.Settings.DirTablebases;
SearchLimit limit = SearchLimit.NodesPerMove(1_000);
//limit = SearchLimit.SecondsForAllMoves(60);
//limit = SearchLimit.SecondsForAllMoves(60, 1f);
limit = SearchLimit.SecondsForAllMoves(120, 0.5f);
GameEngineDefCeres engineDefCeres1 = new GameEngineDefCeres("Ceres1", evalDef1, new ParamsSearch(), null, new ParamsSelect(), null);
GameEngineDefCeres engineDefCeres2 = new GameEngineDefCeres("Ceres2", evalDef2, new ParamsSearch(), null, new ParamsSelect(), null);
bool forceDisableSmartPruning = limit.IsNodesLimit;
if (forceDisableSmartPruning)
{
engineDefCeres1.SearchParams.FutilityPruningStopSearchEnabled = false;
engineDefCeres2.SearchParams.FutilityPruningStopSearchEnabled = false;
}
GameEngineDef engineDefEthereal = new GameEngineDefUCI("Etheral", new GameEngineUCISpec("Etheral", ETHERAL_EXE, NUM_THREADS, HASH_SIZE_MB, TB_PATH, uciSetOptionCommands: extraUCI));
GameEngineDef engineDefStockfish11 = new GameEngineDefUCI("SF11", new GameEngineUCISpec("SF11", SF11_EXE, NUM_THREADS, HASH_SIZE_MB, TB_PATH, uciSetOptionCommands: extraUCI));
GameEngineDef engineDefStockfish12 = new GameEngineDefUCI("SF12", new GameEngineUCISpec("SF12", SF12_EXE, NUM_THREADS, HASH_SIZE_MB, TB_PATH, uciSetOptionCommands: extraUCI));
//GameEngineDef engineDefCeresUCI = new GameEngineDefUCI("CeresUCI", new GameEngineUCISpec("CeresUCI", @"c:\dev\ceres\artifacts\release\net5.0\ceres.exe"));
GameEngineDef engineDefCeresUCI = new GameEngineDefCeresUCI("CeresUCI", evalDef1, overrideEXE: @"c:\dev\ceres\artifacts\release\net5.0\ceres.exe");
GameEngineDefLC0 engineDefLC1 = new GameEngineDefLC0("LC0_0", evalDef1, forceDisableSmartPruning, null, null);
GameEngineDefLC0 engineDefLC2 = new GameEngineDefLC0("LC0_1", evalDef2, forceDisableSmartPruning, null, null);
EnginePlayerDef playerCeres1UCI = new EnginePlayerDef(engineDefCeresUCI, limit);
EnginePlayerDef playerCeres2UCI = new EnginePlayerDef(engineDefCeresUCI, limit);
EnginePlayerDef playerCeres1 = new EnginePlayerDef(engineDefCeres1, limit);
EnginePlayerDef playerCeres2 = new EnginePlayerDef(engineDefCeres2, limit);
EnginePlayerDef playerEthereal = new EnginePlayerDef(engineDefEthereal, limit);
EnginePlayerDef playerStockfish11 = new EnginePlayerDef(engineDefStockfish11, limit);
EnginePlayerDef playerStockfish12 = new EnginePlayerDef(engineDefStockfish12, limit);
EnginePlayerDef playerLC0 = new EnginePlayerDef(engineDefLC1, limit);
EnginePlayerDef playerLC1 = new EnginePlayerDef(engineDefLC2, limit);
// def.SearchLimitEngine1 = def.SearchLimitEngine2 = SearchLimit.SecondsForAllMoves(15, 0.25f);
// def.SearchLimitEngine2 = def.SearchLimitEngine2 = SearchLimit.SecondsForAllMoves(15, 0.25f);
//(playerCeres1.EngineDef as GameEngineDefCeres).SearchParams.DrawByRepetitionLookbackPlies = 40;
if (false)
{
// ===============================================================================
SuiteTestDef suiteDef = new SuiteTestDef("Suite", @"\\synology\dev\chess\data\epd\endgame2.epd", playerCeres1, playerCeres2, null);
// suiteDef.MaxNumPositions = 100;
SuiteTestRunner suiteRunner = new SuiteTestRunner(suiteDef);
suiteRunner.Run(1, true, false);
return;
// ===============================================================================
}
// engineDefCeres2.SearchParams.TwofoldDrawEnabled = false;
//engineDefCeres1.SearchParams.TreeReuseEnabled = false;
//engineDefCeres2.SearchParams.TreeReuseEnabled = false;
//engineDefCeres1.SearchParams.FutilityPruningStopSearchEnabled= false;
//engineDefCeres2.SearchParams.FutilityPruningStopSearchEnabled= false;
//engineDefLC0.SearchParamsEmulate.FutilityPruningStopSearchEnabled= false;
//TournamentDef def = new TournamentDef("TOURN", playerCeres1, playerLC0);
TournamentDef def = new TournamentDef("TOURN", playerCeres1, playerLC0);
def.NumGamePairs = 50;
// def.ShowGameMoves = false;
// def.OpeningsFileName = @"HERT_2017\Hert500.pgn";
// def.StartingFEN = "1q6/2n4k/1r1p1pp1/RP1P2p1/2Q1P1P1/2N4P/3K4/8 b - - 8 71";
// def.OpeningsFileName = @"\\synology\dev\chess\data\openings\Drawkiller_500pos_reordered.pgn";//
def.OpeningsFileName = "TCEC19_NoomenSelect.pgn";
// def.OpeningsFileName = "TCEC1819.pgn";
// def.AdjudicationThresholdCentipawns = 500;
// def.AdjudicationThresholdNumMoves = 3;
const int CONCURRENCY = 1;// 15;
TournamentManager runner = new TournamentManager(def, CONCURRENCY);
TournamentResultStats results;
//UCIEngineProcess.VERBOSE = true;
TimingStats stats = new TimingStats();
using (new TimingBlock(stats, TimingBlock.LoggingType.None))
{
results = runner.RunTournament();
}
Console.WriteLine();
Console.WriteLine($"Tournament completed in {stats.ElapsedTimeSecs,8:F2} seconds.");
Console.WriteLine(results.GameOutcomesString);
}
public SuiteTestResult Run(int numConcurrentSuiteThreads = 1, bool outputDetail = true, bool saveCacheWhenDone = true)
{
// Tree reuse is no help, indicate that we won't need it
Def.Engine1Def.SearchParams.TreeReuseEnabled = false;
if (Def.Engine2Def != null)
{
Def.Engine2Def.SearchParams.TreeReuseEnabled = false;
}
// Disable dump for now, the execution parameters are modified
// for the warmup which is confusing because different parameters
// will be chosen for the actual search.
//DumpParams(Def.Output, true);
// Create evaluators
evaluatorSet1 = new NNEvaluatorSet(Def.Engine1Def.EvaluatorDef);
if (Def.Engine2Def != null)
{
evaluatorSet2 = new NNEvaluatorSet(Def.Engine2Def.EvaluatorDef);
}
numConcurrentSuiteThreads = numConcurrentSuiteThreads;
int timerFiredCount = 0;
// TODO: add path automatically
List <EPDEntry> epds = EPDEntry.EPDEntriesInEPDFile(Def.EPDFileName, int.MaxValue);
if (Def.MaxNumPositions == 0)
{
Def.MaxNumPositions = epds.Count;
}
Def.Output.WriteLine();
Def.Output.WriteLine("C1 = " + Def.Engine1Def.EvaluatorDef);
if (Def.RunCeres2Engine)
{
Def.Output.WriteLine("C2 = " + Def.Engine2Def.EvaluatorDef);
}
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine("EX = " + Def.ExternalEngineDef.EngineDef);
}
#if NOT
// To make up for the fact that LZ0 "cheats" by sometimes running over specified number of nodes
// (she seems to always fill the batch even if reached limit), add half a batch extra for Ceres as compensation
if (searchLimitCeres1.Type == SearchLimit.LimitType.NodesPerMove)
{
searchLimitCeres1 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch1.BATCH_SIZE_PRIMARY / 2);
searchLimitCeres2 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch2.BATCH_SIZE_PRIMARY / 2);
}
#endif
//Def.Output.WriteLine($"MAX_CERES_GAME_THREADS {numConcurrentCeresGames} MAX_LEELA_GAME_THREADS {MAX_LEELA_GAME_THREADS}");
// Turn of position reuse if evaluators produce different results
if (Def.RunCeres2Engine && !Def.Engine1Def.EvaluatorDef.NetEvaluationsIdentical(Def.Engine2Def.EvaluatorDef))
{
Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
}
if (Def.RunCeres2Engine && (Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree ||
Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree))
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("\r\nWARNING: REUSE_POSITION_EVALUATIONS_FROM_OTHER_TREE is turned on for one or both evaluators\r\n"
+ "(alternating between the two evaluators). This may cause slight differences in search behavior and speed.\r\n");
Console.ForegroundColor = ConsoleColor.White;
}
Def.Output.WriteLine();
if (Def.MaxNumPositions > epds.Count)
{
Def.MaxNumPositions = epds.Count;
}
epds = epds.GetRange(Def.FirstTestPosition, Def.MaxNumPositions);
int numExternalGameProcesses = 1;
numConcurrentSuiteThreads = Math.Min(Def.MaxNumPositions, numConcurrentSuiteThreads);
if (numConcurrentSuiteThreads > 1)
{
bool evaluator1NonPooled = Def.Engine1Def?.EvaluatorDef != null && Def.Engine1Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;
bool evaluator2NonPooled = Def.Engine2Def?.EvaluatorDef != null && Def.Engine2Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;
if (evaluator1NonPooled || evaluator2NonPooled)
{
throw new Exception("Must use POOLED neural network evaluator when running suites with parallelism");;
}
if (Def.ExternalEngineDef != null)
{
// For safety (to not overflow main or GPU memory) we limit number of LC0 processes.
const int MAX_LC0_PROCESSES = 4;
numExternalGameProcesses = Math.Min(MAX_LC0_PROCESSES, numConcurrentSuiteThreads);
}
}
bool leelaVerboseMovesStats = true;//xxx Def.NumTestPos == 1;
Func <object> makeExternalEngine = null;
if (Def.ExternalEngineDef != null)
{
if (Def.ExternalEngineDef.EngineDef is GameEngineDefLC0)
{
GameEngineDefLC0 lc0EngineDef = Def.ExternalEngineDef.EngineDef as GameEngineDefLC0;
bool forceDisableSmartPruning = lc0EngineDef.ForceDisableSmartPruning;
makeExternalEngine = () =>
{
LC0Engine engine = LC0EngineConfigured.GetLC0Engine(null, null, Def.Engine1Def.EvaluatorDef,
NNWeightsFiles.LookupNetworkFile(Def.Engine1Def.EvaluatorDef.Nets[0].Net.NetworkID),
true,
false, leelaVerboseMovesStats, forceDisableSmartPruning,
lc0EngineDef.OverrideEXE, false, lc0EngineDef.ExtraCommandLineArgs);
// WARMUP
engine.AnalyzePositionFromFEN(Position.StartPosition.FEN, SearchLimit.NodesPerMove(1));
return(engine);
};
}
else
{
//bool resetMovesBetweenMoves = !Def.Engine2Def.SearchParams.TreeReuseEnabled;
//bool enableTranpsositions = Def.Engine2Def.SearchParams.Execution.TranspositionMode != TranspositionMode.None;
//bool enableTablebases = Def.Engine2Def.SearchParams.EnableTablebases;
makeExternalEngine = () => Def.ExternalEngineDef.EngineDef.CreateEngine();
}
}
// Don't create too many non_Ceres threads since each one will consume seaprate GPU memory or threads
int maxLeelaThreads = Math.Min(numExternalGameProcesses, numConcurrentSuiteThreads);
ObjectPool <object> externalEnginePool = new ObjectPool <object>(makeExternalEngine, maxLeelaThreads);
using (new TimingBlock("EPDS"))
{
Parallel.For(0, epds.Count,
new ParallelOptions()
{
MaxDegreeOfParallelism = numConcurrentSuiteThreads
},
delegate(int gameNum)
{
try
{
EPDEntry epd = epds[gameNum];
// Skip positions which are already draws
if (epd.Position.CheckDrawBasedOnMaterial == Position.PositionDrawStatus.DrawByInsufficientMaterial)
{
return;
}
// TODO: also do this for checkmate?
ProcessEPD(gameNum, epds[gameNum], outputDetail, externalEnginePool);
}
catch (Exception exc)
{
Def.Output.WriteLine("Error in ProcessEPD " + exc);
throw exc;
}
});
}
Def.Output.WriteLine();
Def.Output.WriteLine();
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine($"Total {Def.ExternalEngineDef.ID} Time {totalTimeOther,6:F2}");
}
Def.Output.WriteLine($"Total C1 Time {totalTimeCeres1,6:F2}");
if (Def.CeresEngine2Def != null)
{
Def.Output.WriteLine($"Total C2 Time {totalTimeCeres2,6:F2}");
}
Def.Output.WriteLine();
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine($"Avg {Def.ExternalEngineDef.ID} pos/sec {totalNodesOther / totalTimeOther,8:F2}");
}
Def.Output.WriteLine($"Avg Ceres pos/sec {totalNodes1 / totalTimeCeres1,8:F2}");
if (Def.CeresEngine2Def != null)
{
Def.Output.WriteLine($"Avg Ceres2 pos/sec {totalNodes2 / totalTimeCeres2,8:F2}");
}
Def.Output.WriteLine();
Def.Output.WriteLine();
EngineCeres1.Dispose();
EngineCeres2?.Dispose();
EngineExternal?.Dispose();
externalEnginePool.Shutdown(engineObj => (engineObj as LC0Engine).Dispose());
evaluatorSet1.Dispose();
evaluatorSet2?.Dispose();
return(new SuiteTestResult(Def)
{
AvgScore1 = (float)accCeres1 / numSearches,
AvgScore2 = (float)accCeres2 / numSearches,
AvgWScore1 = (float)accWCeres1 / numSearches,
AvgWScore2 = (float)accWCeres2 / numSearches,
AvgScoreLC0 = (float)avgOther / numSearches,
TotalRuntimeLC0 = totalTimeOther,
TotalRuntime1 = totalTimeCeres1,
TotalRuntime2 = totalTimeCeres2,
TotalNodesLC0 = totalNodesOther,
TotalNodes1 = totalNodes1,
TotalNodes2 = totalNodes2
});
}
