public SuiteTestResult Run(int numConcurrentSuiteThreads = 1, bool outputDetail = true, bool saveCacheWhenDone = true)
{
// Tree reuse is no help, indicate that we won't need it
Def.Engine1Def.SearchParams.TreeReuseEnabled = false;
if (Def.Engine2Def != null)
{
Def.Engine2Def.SearchParams.TreeReuseEnabled = false;
}
// Disable dump for now, the execution parameters are modified
// for the warmup which is confusing because different parameters
// will be chosen for the actual search.
//DumpParams(Def.Output, true);
// Create evaluators
evaluatorSet1 = new NNEvaluatorSet(Def.Engine1Def.EvaluatorDef);
if (Def.Engine2Def != null)
{
evaluatorSet2 = new NNEvaluatorSet(Def.Engine2Def.EvaluatorDef);
}
numConcurrentSuiteThreads = numConcurrentSuiteThreads;
int timerFiredCount = 0;
// TODO: add path automatically
List <EPDEntry> epds = EPDEntry.EPDEntriesInEPDFile(Def.EPDFileName, int.MaxValue);
if (Def.MaxNumPositions == 0)
{
Def.MaxNumPositions = epds.Count;
}
Def.Output.WriteLine();
Def.Output.WriteLine("C1 = " + Def.Engine1Def.EvaluatorDef);
if (Def.RunCeres2Engine)
{
Def.Output.WriteLine("C2 = " + Def.Engine2Def.EvaluatorDef);
}
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine("EX = " + Def.ExternalEngineDef.EngineDef);
}
#if NOT
// To make up for the fact that LZ0 "cheats" by sometimes running over specified number of nodes
// (she seems to always fill the batch even if reached limit), add half a batch extra for Ceres as compensation
if (searchLimitCeres1.Type == SearchLimit.LimitType.NodesPerMove)
{
searchLimitCeres1 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch1.BATCH_SIZE_PRIMARY / 2);
searchLimitCeres2 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch2.BATCH_SIZE_PRIMARY / 2);
}
#endif
//Def.Output.WriteLine($"MAX_CERES_GAME_THREADS {numConcurrentCeresGames} MAX_LEELA_GAME_THREADS {MAX_LEELA_GAME_THREADS}");
// Turn of position reuse if evaluators produce different results
if (Def.RunCeres2Engine && !Def.Engine1Def.EvaluatorDef.NetEvaluationsIdentical(Def.Engine2Def.EvaluatorDef))
{
Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
}
if (Def.RunCeres2Engine && (Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree ||
Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree))
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("\r\nWARNING: REUSE_POSITION_EVALUATIONS_FROM_OTHER_TREE is turned on for one or both evaluators\r\n"
+ "(alternating between the two evaluators). This may cause slight differences in search behavior and speed.\r\n");
Console.ForegroundColor = ConsoleColor.White;
}
Def.Output.WriteLine();
if (Def.MaxNumPositions > epds.Count)
{
Def.MaxNumPositions = epds.Count;
}
epds = epds.GetRange(Def.FirstTestPosition, Def.MaxNumPositions);
int numExternalGameProcesses = 1;
numConcurrentSuiteThreads = Math.Min(Def.MaxNumPositions, numConcurrentSuiteThreads);
if (numConcurrentSuiteThreads > 1)
{
bool evaluator1NonPooled = Def.Engine1Def.EvaluatorDef != null && Def.Engine1Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;
bool evaluator2NonPooled = Def.Engine2Def.EvaluatorDef != null && Def.Engine2Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;
if (evaluator1NonPooled || evaluator2NonPooled)
{
throw new Exception("Must use POOLED neural network evaluator when running suites with parallelism");;
}
if (Def.ExternalEngineDef != null)
{
// For safety (to not overflow main or GPU memory) we limit number of LC0 processes.
const int MAX_LC0_PROCESSES = 4;
numExternalGameProcesses = Math.Min(MAX_LC0_PROCESSES, numConcurrentSuiteThreads);
}
}
bool leelaVerboseMovesStats = true;//xxx Def.NumTestPos == 1;
Func <object> makeExternalEngine = null;
if (Def.ExternalEngineDef != null)
{
if (Def.ExternalEngineDef.EngineDef is GameEngineDefLC0)
{
bool forceDisableSmartPruning = (Def.ExternalEngineDef.EngineDef as GameEngineDefLC0).ForceDisableSmartPruning;
makeExternalEngine = () =>
{
LC0Engine engine = LC0EngineConfigured.GetLC0Engine(null, null, Def.Engine1Def.EvaluatorDef,
NNWeightsFiles.LookupNetworkFile(Def.Engine1Def.EvaluatorDef.Nets[0].Net.NetworkID),
true,
false, leelaVerboseMovesStats, forceDisableSmartPruning);
// WARMUP
engine.AnalyzePositionFromFEN(Position.StartPosition.FEN, null, SearchLimit.NodesPerMove(1));
return(engine);
};
}
else
{
bool resetMovesBetweenMoves = !Def.Engine2Def.SearchParams.TreeReuseEnabled;
bool enableTranpsositions = Def.Engine2Def.SearchParams.Execution.TranspositionMode != TranspositionMode.None;
bool enableTablebases = Def.Engine2Def.SearchParams.EnableTablebases;
makeExternalEngine = () => Def.ExternalEngineDef.EngineDef.CreateEngine();
}
}
// Don't create too many non_Ceres threads since each one will consume seaprate GPU memory or threads
int maxLeelaThreads = Math.Min(numExternalGameProcesses, numConcurrentSuiteThreads);
ObjectPool <object> externalEnginePool = new ObjectPool <object>(makeExternalEngine, maxLeelaThreads);
using (new TimingBlock("EPDS"))
{
Parallel.For(0, epds.Count,
new ParallelOptions()
{
MaxDegreeOfParallelism = numConcurrentSuiteThreads
},
delegate(int gameNum)
{
try
{
EPDEntry epd = epds[gameNum];
// Skip positions which are already draws
if (epd.Position.CheckDrawBasedOnMaterial == Position.PositionDrawStatus.DrawByInsufficientMaterial)
{
return;
}
// TODO: also do this for checkmate?
ProcessEPD(gameNum, epds[gameNum], outputDetail, externalEnginePool);
}
catch (Exception exc)
{
Def.Output.WriteLine("Error in ProcessEPD " + exc);
throw exc;
}
});
}
Def.Output.WriteLine();
Def.Output.WriteLine();
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine($"Total {Def.ExternalEngineDef.ID} Time {totalTimeOther,6:F2}");
}
Def.Output.WriteLine($"Total C1 Time {totalTimeCeres1,6:F2}");
if (Def.CeresEngine2Def != null)
{
Def.Output.WriteLine($"Total C2 Time {totalTimeCeres2,6:F2}");
}
Def.Output.WriteLine();
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine($"Avg {Def.ExternalEngineDef.ID} pos/sec {totalNodesOther / totalTimeOther,8:F2}");
}
Def.Output.WriteLine($"Avg Ceres pos/sec {totalNodes1 / totalTimeCeres1,8:F2}");
if (Def.CeresEngine2Def != null)
{
Def.Output.WriteLine($"Avg Ceres2 pos/sec {totalNodes2 / totalTimeCeres2,8:F2}");
}
Def.Output.WriteLine();
Def.Output.WriteLine();
EngineCeres1.Dispose();
EngineCeres2?.Dispose();
EngineExternal?.Dispose();
externalEnginePool.Shutdown(engineObj => (engineObj as LC0Engine).Dispose());
evaluatorSet1.Dispose();
evaluatorSet2?.Dispose();
return(new SuiteTestResult(Def)
{
AvgScore1 = (float)accCeres1 / numSearches,
AvgScore2 = (float)accCeres2 / numSearches,
AvgWScore1 = (float)accWCeres1 / numSearches,
AvgWScore2 = (float)accWCeres2 / numSearches,
AvgScoreLC0 = (float)avgOther / numSearches,
TotalRuntimeLC0 = totalTimeOther,
TotalRuntime1 = totalTimeCeres1,
TotalRuntime2 = totalTimeCeres2,
TotalNodesLC0 = totalNodesOther,
TotalNodes1 = totalNodes1,
TotalNodes2 = totalNodes2
});
}
/// <summary>
/// Runs the UCI loop.
/// </summary>
public void PlayUCI()
{
// Default to the startpos.
curPositionAndMoves = PositionWithHistory.FromFENAndMovesUCI(Position.StartPosition.FEN, null);
curManager = null;
gameMoveHistory = new List <GameMoveStat>();
while (true)
{
string command = InStream.ReadLine();
switch (command)
{
case null:
case "":
break;
case "uci":
Send("id name Ceres"); // TODO: Add executable version
Send("id author David Elliott and the Ceres Authors");
// todo output options such as:
// option name Logfile type check default false
Send("uciok");
break;
case "setoption":
OutStream.WriteLine("Not processing option " + command);
return;
case "stop":
if (taskSearchCurrentlyExecuting != null && !stopIsPending)
{
stopIsPending = true;
// TODO: cleanup
// Possible race condition, curManager is only set in search callback which may not have hit yet
// Fix eventually by rewriting SerchManager to have a constructor and then non-static Search method,
// os we can get the context in this class directly after construction
while (curManager == null)
{
Thread.Sleep(1); // **** TEMPORARY ***
}
curManager.ExternalStopRequested = true;
if (taskSearchCurrentlyExecuting != null)
{
taskSearchCurrentlyExecuting.Wait();
if (!debug && taskSearchCurrentlyExecuting != null)
{
taskSearchCurrentlyExecuting.Result?.Manager?.Dispose();
}
taskSearchCurrentlyExecuting = null;
}
}
curManager = null;
stopIsPending = false;
break;
case "ponderhit":
throw new NotImplementedException("Ceres does not yet support UCI ponder mode.");
return;
case "xboard":
// ignore
break;
case "debug on":
debug = true;
break;
case "debug off":
debug = false;
break;
case "isready":
InitializeEngineIfNeeded();
Send("readyok");
break;
case "ucinewgame":
gameMoveHistory = new List <GameMoveStat>();
CeresEngine?.ResetGame();
break;
case "quit":
if (curManager != null)
{
curManager.ExternalStopRequested = true;
taskSearchCurrentlyExecuting?.Wait();
}
if (CeresEngine != null)
{
CeresEngine.Dispose();
}
System.Environment.Exit(0);
break;
case string c when c.StartsWith("go"):
if (taskSearchCurrentlyExecuting != null)
{
throw new Exception("Received go command when another search was running and not stopped first");
}
InitializeEngineIfNeeded();
taskSearchCurrentlyExecuting = ProcessGo(command);
break;
case string c when c.StartsWith("position"):
try
{
ProcessPosition(c);
}
catch (Exception e)
{
Send($"Illegal position command: \"{c}\"" + System.Environment.NewLine + e.ToString());
}
break;
// Proprietary commands
case "lc0-config":
if (curManager != null)
{
string netID = EvaluatorDef.Nets[0].Net.NetworkID;
INNWeightsFileInfo netDef = NNWeightsFiles.LookupNetworkFile(netID);
(string exe, string options) = LC0EngineConfigured.GetLC0EngineOptions(null, null, curContext.EvaluatorDef, netDef, false, false);
Console.WriteLine("info string " + exe + " " + options);
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-params":
if (curManager != null)
{
curManager.DumpParams();
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-processor":
HardwareManager.DumpProcessorInfo();
break;
case "dump-time":
if (curManager != null)
{
curManager.DumpTimeInfo();
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-store":
if (curManager != null)
{
using (new SearchContextExecutionBlock(curContext))
curManager.Context.Tree.Store.Dump(true);
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-move-stats":
if (curManager != null)
{
using (new SearchContextExecutionBlock(curContext))
curManager.Context.Root.Dump(1, 1, prefixString: "info string ");
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-pv":
DumpPV(false);
break;
case "dump-pv-detail":
DumpPV(true);
break;
case "dump-nvidia":
NVML.DumpInfo();
break;
case "waitdone": // proprietary verb
taskSearchCurrentlyExecuting?.Wait();
break;
default:
Console.WriteLine($"error Unknown command: {command}");
break;
}
}
}
/// <summary>
/// Runs the UCI loop.
/// </summary>
public void PlayUCI()
{
// Default to the startpos.
curPositionAndMoves = PositionWithHistory.FromFENAndMovesUCI(Position.StartPosition.FEN);
gameMoveHistory = new List <GameMoveStat>();
while (true)
{
string command = InStream.ReadLine();
if (uciLogWriter != null)
{
LogWriteLine("IN:", command);
}
switch (command)
{
case null:
case "":
break;
case "uci":
UCIWriteLine($"id name Ceres {CeresVersion.VersionString}");
UCIWriteLine("id author David Elliott and the Ceres Authors");
UCIWriteLine(SetOptionUCIDescriptions);
UCIWriteLine("uciok");
break;
case string c when c.StartsWith("setoption"):
ProcessSetOption(command);
break;
case "stop":
if (taskSearchCurrentlyExecuting != null && !stopIsPending)
{
stopIsPending = true;
// Avoid race condition by mkaing sure the search is already created.
while (CeresEngine.Search?.Manager == null)
{
Thread.Sleep(20);
}
CeresEngine.Search.Manager.ExternalStopRequested = true;
if (taskSearchCurrentlyExecuting != null)
{
taskSearchCurrentlyExecuting.Wait();
// if (!debug && taskSearchCurrentlyExecuting != null) taskSearchCurrentlyExecuting.Result?.Search?.Manager?.Dispose();
taskSearchCurrentlyExecuting = null;
}
}
stopIsPending = false;
break;
case "ponderhit":
throw new NotImplementedException("Ceres does not yet support UCI ponder mode.");
return;
case "xboard":
// ignore
break;
case "debug on":
debug = true;
break;
case "debug off":
debug = false;
break;
case "isready":
InitializeEngineIfNeeded();
UCIWriteLine("readyok");
break;
case "ucinewgame":
gameMoveHistory = new List <GameMoveStat>();
CeresEngine?.ResetGame();
break;
case "quit":
if (taskSearchCurrentlyExecuting != null)
{
CeresEngine.Search.Manager.ExternalStopRequested = true;
taskSearchCurrentlyExecuting?.Wait();
}
if (CeresEngine != null)
{
CeresEngine.Dispose();
}
System.Environment.Exit(0);
break;
case string c when c.StartsWith("go"):
// Possibly another search is already executing.
// The UCI specification is unclear about what to do in this situation.
// Some engines seem to enqueue these for later execution (e.g. Stockfish)
// whereas others (e.g. Python chess) report this as an error condition.
// Currently Ceres waits only a short while for any possible pending search
// to finish (e.g. to avoid a race condition if it is in the process of being shutdown)
// and aborts with an error if search is still in progress.
// It is not viable to wait indefinitely, since (among other reasons)
// the engine needs to monitor for stop commands.
const int MAX_MILLISECONDS_WAIT = 500;
taskSearchCurrentlyExecuting?.Wait(MAX_MILLISECONDS_WAIT);
if (taskSearchCurrentlyExecuting != null && !taskSearchCurrentlyExecuting.IsCompleted)
{
throw new Exception("Received go command when another search was running and not stopped first.");
}
InitializeEngineIfNeeded();
taskSearchCurrentlyExecuting = ProcessGo(command);
break;
case string c when c.StartsWith("position"):
try
{
ProcessPosition(c);
}
catch (Exception e)
{
UCIWriteLine($"Illegal position command: \"{c}\"" + System.Environment.NewLine + e.ToString());
}
break;
// Proprietary commands
case "lc0-config":
if (CeresEngine?.Search != null)
{
string netID = EvaluatorDef.Nets[0].Net.NetworkID;
INNWeightsFileInfo netDef = NNWeightsFiles.LookupNetworkFile(netID);
(string exe, string options) = LC0EngineConfigured.GetLC0EngineOptions(null, null, CeresEngine.Search.Manager.Context.EvaluatorDef, netDef, false, false);
UCIWriteLine("info string " + exe + " " + options);
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-params":
if (CeresEngine?.Search != null)
{
CeresEngine?.Search.Manager.DumpParams();
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-processor":
HardwareManager.DumpProcessorInfo();
break;
case "dump-time":
if (CeresEngine?.Search != null)
{
CeresEngine?.Search.Manager.DumpTimeInfo(OutStream);
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-store":
if (CeresEngine?.Search != null)
{
using (new SearchContextExecutionBlock(CeresEngine.Search.Manager.Context))
CeresEngine.Search.Manager.Context.Tree.Store.Dump(true);
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-move-stats":
if (CeresEngine?.Search != null)
{
OutputVerboseMoveStats(CeresEngine.Search.SearchRootNode);
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-pv":
DumpPV(false);
break;
case "dump-pv-detail":
DumpPV(true);
break;
case "dump-nvidia":
NVML.DumpInfo();
break;
case "show-tree-plot":
if (CeresEngine?.Search != null)
{
using (new SearchContextExecutionBlock(CeresEngine.Search.Manager.Context))
{
TreePlot.Show(CeresEngine.Search.Manager.Context.Root.Ref);
}
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case string c when c.StartsWith("save-tree-plot"):
if (CeresEngine?.Search != null)
{
string[] parts = command.Split(" ");
if (parts.Length == 2)
{
string fileName = parts[1];
using (new SearchContextExecutionBlock(CeresEngine.Search.Manager.Context))
{
TreePlot.Save(CeresEngine.Search.Manager.Context.Root.Ref, fileName);
}
}
else if (parts.Length == 1)
{
UCIWriteLine("Filename was not provided");
}
else
{
UCIWriteLine("Filename cannot contain spaces");
}
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "waitdone": // proprietary verb used for test driver
taskSearchCurrentlyExecuting?.Wait();
break;
default:
UCIWriteLine($"error Unknown command: {command}");
break;
}
}
}
