/// <summary>
/// Parses a specification of search time in UCI format into an equivalent SearchLimit.
/// Returns null if parsing failed.
/// </summary>
/// <param name="command"></param>
/// <returns></returns>
private SearchLimit GetSearchLimit(string command)
{
bool weAreWhite = curPositionAndMoves.FinalPosition.MiscInfo.SideToMove == SideType.White;
UCIGoCommandParsed goInfo = new UCIGoCommandParsed(command, weAreWhite);
if (!goInfo.IsValid)
{
return(null);
}
if (goInfo.Nodes.HasValue)
{
return(SearchLimit.NodesPerMove(goInfo.Nodes.Value));
}
else if (goInfo.MoveTime.HasValue)
{
return(SearchLimit.SecondsPerMove(goInfo.MoveTime.Value / 1000.0f));
}
else if (goInfo.Infinite)
{
// TODO: determine a reasonable maximum value based on memory in computer
return(SearchLimit.NodesPerMove(MCTSNodeStore.MAX_NODES));
}
else if (goInfo.TimeOurs.HasValue)
{
float increment = 0;
if (goInfo.IncrementOurs.HasValue)
{
increment = goInfo.IncrementOurs.Value / 1000.0f;
}
int?movesToGo = null;
if (goInfo.MovesToGo.HasValue)
{
movesToGo = goInfo.MovesToGo.Value;
}
return(SearchLimit.SecondsForAllMoves(goInfo.TimeOurs.Value / 1000.0f, increment, movesToGo, true));
}
else
{
Console.WriteLine($"Unsupported time control in UCI go command {command}");
return(null);
}
}
public SuiteTestResult Run(int numConcurrentSuiteThreads = 1, bool outputDetail = true, bool saveCacheWhenDone = true)
{
// Tree reuse is no help, indicate that we won't need it
Def.Engine1Def.SearchParams.TreeReuseEnabled = false;
if (Def.Engine2Def != null)
{
Def.Engine2Def.SearchParams.TreeReuseEnabled = false;
}
// Disable dump for now, the execution parameters are modified
// for the warmup which is confusing because different parameters
// will be chosen for the actual search.
//DumpParams(Def.Output, true);
// Create evaluators
evaluatorSet1 = new NNEvaluatorSet(Def.Engine1Def.EvaluatorDef);
if (Def.Engine2Def != null)
{
evaluatorSet2 = new NNEvaluatorSet(Def.Engine2Def.EvaluatorDef);
}
numConcurrentSuiteThreads = numConcurrentSuiteThreads;
int timerFiredCount = 0;
// TODO: add path automatically
List <EPDEntry> epds = EPDEntry.EPDEntriesInEPDFile(Def.EPDFileName, int.MaxValue);
if (Def.MaxNumPositions == 0)
{
Def.MaxNumPositions = epds.Count;
}
Def.Output.WriteLine();
Def.Output.WriteLine("C1 = " + Def.Engine1Def.EvaluatorDef);
if (Def.RunCeres2Engine)
{
Def.Output.WriteLine("C2 = " + Def.Engine2Def.EvaluatorDef);
}
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine("EX = " + Def.ExternalEngineDef.EngineDef);
}
#if NOT
// To make up for the fact that LZ0 "cheats" by sometimes running over specified number of nodes
// (she seems to always fill the batch even if reached limit), add half a batch extra for Ceres as compensation
if (searchLimitCeres1.Type == SearchLimit.LimitType.NodesPerMove)
{
searchLimitCeres1 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch1.BATCH_SIZE_PRIMARY / 2);
searchLimitCeres2 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch2.BATCH_SIZE_PRIMARY / 2);
}
#endif
//Def.Output.WriteLine($"MAX_CERES_GAME_THREADS {numConcurrentCeresGames} MAX_LEELA_GAME_THREADS {MAX_LEELA_GAME_THREADS}");
// Turn of position reuse if evaluators produce different results
if (Def.RunCeres2Engine && !Def.Engine1Def.EvaluatorDef.NetEvaluationsIdentical(Def.Engine2Def.EvaluatorDef))
{
Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
}
if (Def.RunCeres2Engine && (Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree ||
Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree))
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("\r\nWARNING: REUSE_POSITION_EVALUATIONS_FROM_OTHER_TREE is turned on for one or both evaluators\r\n"
+ "(alternating between the two evaluators). This may cause slight differences in search behavior and speed.\r\n");
Console.ForegroundColor = ConsoleColor.White;
}
Def.Output.WriteLine();
if (Def.MaxNumPositions > epds.Count)
{
Def.MaxNumPositions = epds.Count;
}
epds = epds.GetRange(Def.FirstTestPosition, Def.MaxNumPositions);
int numExternalGameProcesses = 1;
numConcurrentSuiteThreads = Math.Min(Def.MaxNumPositions, numConcurrentSuiteThreads);
if (numConcurrentSuiteThreads > 1)
{
bool evaluator1NonPooled = Def.Engine1Def.EvaluatorDef != null && Def.Engine1Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;
bool evaluator2NonPooled = Def.Engine2Def.EvaluatorDef != null && Def.Engine2Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;
if (evaluator1NonPooled || evaluator2NonPooled)
{
throw new Exception("Must use POOLED neural network evaluator when running suites with parallelism");;
}
if (Def.ExternalEngineDef != null)
{
// For safety (to not overflow main or GPU memory) we limit number of LC0 processes.
const int MAX_LC0_PROCESSES = 4;
numExternalGameProcesses = Math.Min(MAX_LC0_PROCESSES, numConcurrentSuiteThreads);
}
}
bool leelaVerboseMovesStats = true;//xxx Def.NumTestPos == 1;
Func <object> makeExternalEngine = null;
if (Def.ExternalEngineDef != null)
{
if (Def.ExternalEngineDef.EngineDef is GameEngineDefLC0)
{
bool forceDisableSmartPruning = (Def.ExternalEngineDef.EngineDef as GameEngineDefLC0).ForceDisableSmartPruning;
makeExternalEngine = () =>
{
LC0Engine engine = LC0EngineConfigured.GetLC0Engine(null, null, Def.Engine1Def.EvaluatorDef,
NNWeightsFiles.LookupNetworkFile(Def.Engine1Def.EvaluatorDef.Nets[0].Net.NetworkID),
true,
false, leelaVerboseMovesStats, forceDisableSmartPruning);
// WARMUP
engine.AnalyzePositionFromFEN(Position.StartPosition.FEN, null, SearchLimit.NodesPerMove(1));
return(engine);
};
}
else
{
bool resetMovesBetweenMoves = !Def.Engine2Def.SearchParams.TreeReuseEnabled;
bool enableTranpsositions = Def.Engine2Def.SearchParams.Execution.TranspositionMode != TranspositionMode.None;
bool enableTablebases = Def.Engine2Def.SearchParams.EnableTablebases;
makeExternalEngine = () => Def.ExternalEngineDef.EngineDef.CreateEngine();
}
}
// Don't create too many non_Ceres threads since each one will consume seaprate GPU memory or threads
int maxLeelaThreads = Math.Min(numExternalGameProcesses, numConcurrentSuiteThreads);
ObjectPool <object> externalEnginePool = new ObjectPool <object>(makeExternalEngine, maxLeelaThreads);
using (new TimingBlock("EPDS"))
{
Parallel.For(0, epds.Count,
new ParallelOptions()
{
MaxDegreeOfParallelism = numConcurrentSuiteThreads
},
delegate(int gameNum)
{
try
{
EPDEntry epd = epds[gameNum];
// Skip positions which are already draws
if (epd.Position.CheckDrawBasedOnMaterial == Position.PositionDrawStatus.DrawByInsufficientMaterial)
{
return;
}
// TODO: also do this for checkmate?
ProcessEPD(gameNum, epds[gameNum], outputDetail, externalEnginePool);
}
catch (Exception exc)
{
Def.Output.WriteLine("Error in ProcessEPD " + exc);
throw exc;
}
});
}
Def.Output.WriteLine();
Def.Output.WriteLine();
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine($"Total {Def.ExternalEngineDef.ID} Time {totalTimeOther,6:F2}");
}
Def.Output.WriteLine($"Total C1 Time {totalTimeCeres1,6:F2}");
if (Def.CeresEngine2Def != null)
{
Def.Output.WriteLine($"Total C2 Time {totalTimeCeres2,6:F2}");
}
Def.Output.WriteLine();
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine($"Avg {Def.ExternalEngineDef.ID} pos/sec {totalNodesOther / totalTimeOther,8:F2}");
}
Def.Output.WriteLine($"Avg Ceres pos/sec {totalNodes1 / totalTimeCeres1,8:F2}");
if (Def.CeresEngine2Def != null)
{
Def.Output.WriteLine($"Avg Ceres2 pos/sec {totalNodes2 / totalTimeCeres2,8:F2}");
}
Def.Output.WriteLine();
Def.Output.WriteLine();
EngineCeres1.Dispose();
EngineCeres2?.Dispose();
EngineExternal?.Dispose();
externalEnginePool.Shutdown(engineObj => (engineObj as LC0Engine).Dispose());
evaluatorSet1.Dispose();
evaluatorSet2?.Dispose();
return(new SuiteTestResult(Def)
{
AvgScore1 = (float)accCeres1 / numSearches,
AvgScore2 = (float)accCeres2 / numSearches,
AvgWScore1 = (float)accWCeres1 / numSearches,
AvgWScore2 = (float)accWCeres2 / numSearches,
AvgScoreLC0 = (float)avgOther / numSearches,
TotalRuntimeLC0 = totalTimeOther,
TotalRuntime1 = totalTimeCeres1,
TotalRuntime2 = totalTimeCeres2,
TotalNodesLC0 = totalNodesOther,
TotalNodes1 = totalNodes1,
TotalNodes2 = totalNodes2
});
}
/// <summary>
/// Parses a specification of search time in UCI format into an equivalent SearchLimit.
/// Returns null if parsing failed.
/// </summary>
/// <param name="command"></param>
/// <returns></returns>
private SearchLimit GetSearchLimit(string command)
{
SearchLimit searchLimit;
bool weAreWhite = curPositionAndMoves.FinalPosition.MiscInfo.SideToMove == SideType.White;
UCIGoCommandParsed goInfo = new UCIGoCommandParsed(command, weAreWhite);
if (!goInfo.IsValid)
{
return(null);
}
if (goInfo.Nodes.HasValue)
{
searchLimit = SearchLimit.NodesPerMove(goInfo.Nodes.Value);
}
else if (goInfo.MoveTime.HasValue)
{
searchLimit = SearchLimit.SecondsPerMove(goInfo.MoveTime.Value / 1000.0f);
}
else if (goInfo.Infinite)
{
searchLimit = SearchLimit.NodesPerMove(MCTSNodeStore.MAX_NODES);
}
else if (goInfo.TimeOurs.HasValue)
{
float increment = 0;
if (goInfo.IncrementOurs.HasValue)
{
increment = goInfo.IncrementOurs.Value / 1000.0f;
}
int?movesToGo = null;
if (goInfo.MovesToGo.HasValue)
{
movesToGo = goInfo.MovesToGo.Value;
}
searchLimit = SearchLimit.SecondsForAllMoves(goInfo.TimeOurs.Value / 1000.0f, increment, movesToGo, true);
}
else if (goInfo.NodesOurs.HasValue)
{
float increment = 0;
if (goInfo.IncrementOurs.HasValue)
{
increment = goInfo.IncrementOurs.Value;
}
int?movesToGo = null;
if (goInfo.MovesToGo.HasValue)
{
movesToGo = goInfo.MovesToGo.Value;
}
searchLimit = SearchLimit.NodesForAllMoves(goInfo.NodesOurs.Value, (int)increment, movesToGo, true);
}
else
{
UCIWriteLine($"Unsupported time control in UCI go command {command}");
return(null);
}
// Add on possible search moves restriction.
return(searchLimit with {
SearchMoves = goInfo.SearchMoves
});
}
/// <summary>
/// Dump two sets of parameters, optionally showing only differences.
/// </summary>
public static void DumpParams(TextWriter writer, bool differentOnly,
GameEngineUCISpec externalEngine1Spec, GameEngineUCISpec externalEngine2Spec,
NNEvaluatorDef evaluatorDef1, NNEvaluatorDef evaluatorDef2,
SearchLimit searchLimit1, SearchLimit searchLimit2,
ParamsSelect selectParams1, ParamsSelect selectParams2,
ParamsSearch searchParams1, ParamsSearch searchParams2,
IManagerGameLimit timeManager1, IManagerGameLimit timeManager2,
ParamsSearchExecution paramsSearchExecution1,
ParamsSearchExecution paramsSearchExecution2)
{
writer.WriteLine("\r\n-----------------------------------------------------------------------");
writer.WriteLine("ENGINE 1 Options Modifications from Default");
if (evaluatorDef1 != null)
{
writer.WriteLine("Ceres Evaluator : " + evaluatorDef1.ToString());
}
else
{
writer.Write("External UCI : " + externalEngine1Spec);
}
writer.Write(ObjUtils.FieldValuesDumpString <SearchLimit>(searchLimit1, SearchLimit.NodesPerMove(1), differentOnly));
// writer.Write(ObjUtils.FieldValuesDumpString<NNEvaluatorDef>(Def.NNEvaluators1.EvaluatorDef, new ParamsNN(), differentOnly));
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSelect>(selectParams1, new ParamsSelect(), differentOnly));
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSearch>(searchParams1, new ParamsSearch(), differentOnly));
DumpTimeManagerDifference(true, null, timeManager1);
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSearchExecution>(paramsSearchExecution1, new ParamsSearchExecution(), differentOnly));
writer.WriteLine("\r\n-----------------------------------------------------------------------");
writer.WriteLine("ENGINE 2 Options Modifications from Engine 1");
bool evaluatorsDifferent = false;
bool eval2TypeDifferent = (evaluatorDef1 == null) != (evaluatorDef2 == null);
if (eval2TypeDifferent)
{
evaluatorsDifferent = true;
}
else
{
if (evaluatorDef1 != null)
{
evaluatorsDifferent = evaluatorDef1.ToString() != evaluatorDef2.ToString();
}
else
{
evaluatorsDifferent = externalEngine1Spec.ToString() != externalEngine2Spec.ToString();
}
}
if (!differentOnly || evaluatorsDifferent)
{
if (evaluatorDef1 != null && evaluatorDef2 != null)
{
writer.WriteLine("Ceres Evaluator : " + evaluatorDef2.ToString());
}
else
{
writer.Write("External UCI : " + externalEngine2Spec);
}
}
if (searchLimit2 != null)
{
writer.Write(ObjUtils.FieldValuesDumpString <SearchLimit>(searchLimit2, searchLimit1, differentOnly));
// writer.Write(ObjUtils.FieldValuesDumpString<NNEvaluatorDef>(Def.NNEvaluators1.EvaluatorDef, Def.NNEvaluators2.EvaluatorDef, differentOnly));
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSelect>(selectParams2, selectParams1, differentOnly));
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSearch>(searchParams2, searchParams1, differentOnly));
DumpTimeManagerDifference(differentOnly, timeManager1, timeManager2);
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSearchExecution>(paramsSearchExecution2, paramsSearchExecution1, differentOnly));
}
}
