/// <summary>
///
/// </summary>
/// <param name="store"></param>
/// <param name="reuseOtherContextForEvaluatedNodes"></param>
/// <param name="reusePositionCache"></param>
/// <param name="reuseTranspositionRoots"></param>
/// <param name="nnEvaluators"></param>
/// <param name="searchParams"></param>
/// <param name="childSelectParams"></param>
/// <param name="searchLimit"></param>
/// <param name="paramsSearchExecutionPostprocessor"></param>
/// <param name="limitManager"></param>
/// <param name="startTime"></param>
/// <param name="priorManager"></param>
/// <param name="gameMoveHistory"></param>
/// <param name="isFirstMoveOfGame"></param>
public MCTSManager(MCTSNodeStore store,
MCTSIterator reuseOtherContextForEvaluatedNodes,
PositionEvalCache reusePositionCache,
TranspositionRootsDict reuseTranspositionRoots,
NNEvaluatorSet nnEvaluators,
ParamsSearch searchParams,
ParamsSelect childSelectParams,
SearchLimit searchLimit,
ParamsSearchExecutionModifier paramsSearchExecutionPostprocessor,
IManagerGameLimit limitManager,
DateTime startTime,
MCTSManager priorManager,
List <GameMoveStat> gameMoveHistory,
bool isFirstMoveOfGame)
{
StartTimeThisSearch = startTime;
RootNWhenSearchStarted = store.Nodes.nodes[store.RootIndex.Index].N;
ParamsSearchExecutionPostprocessor = paramsSearchExecutionPostprocessor;
IsFirstMoveOfGame = isFirstMoveOfGame;
PriorMoveStats = new List <GameMoveStat>();
// Make our own copy of move history.
if (gameMoveHistory != null)
{
PriorMoveStats.AddRange(gameMoveHistory);
}
// Possibly convert time limit per game into time for this move.
if (searchLimit.IsPerGameLimit)
{
SearchLimitType type = searchLimit.Type == SearchLimitType.SecondsForAllMoves
? SearchLimitType.SecondsPerMove
: SearchLimitType.NodesPerMove;
float rootQ = priorManager == null ? float.NaN : (float)store.RootNode.Q;
limitsManagerInputs = new(store.Nodes.PriorMoves.FinalPosition,
searchParams, PriorMoveStats,
type, store.RootNode.N, rootQ,
searchLimit.Value, searchLimit.ValueIncrement,
float.NaN, float.NaN,
maxMovesToGo : searchLimit.MaxMovesToGo,
isFirstMoveOfGame : isFirstMoveOfGame);
ManagerGameLimitOutputs timeManagerOutputs = limitManager.ComputeMoveAllocation(limitsManagerInputs);
SearchLimit = timeManagerOutputs.LimitTarget;
}
else
{
SearchLimit = searchLimit;
}
// Possibly autoselect new optimal parameters
ParamsSearchExecutionChooser paramsChooser = new ParamsSearchExecutionChooser(nnEvaluators.EvaluatorDef,
searchParams, childSelectParams, searchLimit);
// TODO: technically this is overwriting the params belonging to the prior search, that's ugly (but won't actually cause a problem)
paramsChooser.ChooseOptimal(searchLimit.EstNumNodes(50_000, false), paramsSearchExecutionPostprocessor); // TODO: make 50_000 smarter
int estNumNodes = EstimatedNumSearchNodesForEvaluator(searchLimit, nnEvaluators);
// Adjust the nodes estimate if we are continuing an existing search
if (searchLimit.Type == SearchLimitType.NodesPerMove && RootNWhenSearchStarted > 0)
{
estNumNodes = Math.Max(0, estNumNodes - RootNWhenSearchStarted);
}
Context = new MCTSIterator(store, reuseOtherContextForEvaluatedNodes, reusePositionCache, reuseTranspositionRoots,
nnEvaluators, searchParams, childSelectParams, searchLimit, estNumNodes);
ThreadSearchContext = Context;
TerminationManager = new MCTSFutilityPruning(this, Context);
LimitManager = limitManager;
CeresEnvironment.LogInfo("MCTS", "Init", $"SearchManager created for store {store}", InstanceID);
}
public static void Analyze(string fen, SearchLimit searchLimit,
NNEvaluatorDef evaluatorDef,
bool forceDisablePruning,
LC0Engine lc0Engine = null,
GameEngine comparisonEngine = null,
bool verbose = false)
{
Console.WriteLine("=============================================================================");
Console.WriteLine("Analyzing FEN : " + fen);
Console.WriteLine("Search limit : " + searchLimit.ToString());
Console.WriteLine("Ceres evaluator : " + evaluatorDef.ToString());
if (comparisonEngine != null)
{
Console.WriteLine("Opponent : " + comparisonEngine.ToString());
}
Console.WriteLine();
Console.WriteLine();
NNEvaluatorSet nnEvaluators = new NNEvaluatorSet(evaluatorDef);
// Warmup (in parallel)
lc0Engine?.DoSearchPrepare();
Parallel.Invoke(
() => nnEvaluators.Warmup(true),
() => comparisonEngine?.Warmup());
bool ceresDone = false;
lastInfoUpdate = DateTime.Now;
UCISearchInfo lastCeresInfo = null;
// Launch Ceres
MCTSearch ceresResults = null;
Task searchCeres = Task.Run(() =>
{
ParamsSearch searchParams = new ParamsSearch();
searchParams.FutilityPruningStopSearchEnabled = !forceDisablePruning;
PositionWithHistory positionWithHistory = PositionWithHistory.FromFENAndMovesUCI(fen, null);
ceresResults = new MCTSearch();
ceresResults.Search(nnEvaluators, new ParamsSelect(), searchParams, null, null,
null, positionWithHistory, searchLimit, verbose, DateTime.Now, null,
manager => lastCeresInfo = new UCISearchInfo(UCIManager.UCIInfoString(manager), null, null), false, true);
});
// Possibly launch search for other engine
Task searchComparison = null;
if (lc0Engine != null || comparisonEngine != null)
{
searchComparison = Task.Run(() =>
{
if (lc0Engine != null)
{
lc0Engine.DoSearchPrepare();
lc0Engine.AnalyzePositionFromFENAndMoves(fen, null, fen, searchLimit);
}
else
{
// TODO: someday enable passing in of moves here
comparisonEngine.Search(PositionWithHistory.FromFENAndMovesUCI(fen, null), searchLimit, verbose: true);
}
});
}
;
while (!searchCeres.IsCompleted || (searchComparison != null && !searchComparison.IsCompleted))
{
Thread.Sleep(1000);
//Console.WriteLine(DateTime.Now + " --> " + lastCeresInfo?.PVString + " OTHER " + comparisonEngine?.UCIInfo?.RawString);
int numCharactersSame = int.MaxValue;
if (lastCeresInfo?.PVString != null || comparisonEngine?.UCIInfo?.RawString != null)
{
if (lastCeresInfo != null && comparisonEngine?.UCIInfo != null)
{
numCharactersSame = 0;
string pv1 = lastCeresInfo.PVString;
UCISearchInfo lastComparisonInfo = comparisonEngine.UCIInfo;
string pv2 = lastComparisonInfo.PVString;
while (pv1.Length > numCharactersSame &&
pv2.Length > numCharactersSame &&
pv1[numCharactersSame] == pv2[numCharactersSame])
{
numCharactersSame++;
}
}
}
if (lastCeresInfo != null)
{
WriteUCI("Ceres", lastCeresInfo, numCharactersSame);
}
if (comparisonEngine != null)
{
WriteUCI(comparisonEngine.ID, comparisonEngine.UCIInfo, numCharactersSame);
}
Console.WriteLine();
}
searchCeres.Wait();
searchComparison?.Wait();
string infoUpdate = UCIManager.UCIInfoString(ceresResults.Manager);
double q2 = ceresResults.BestMoveRoot.Q;
//SearchPrincipalVariation pv2 = new SearchPrincipalVariation(worker2.Root);
MCTSPosTreeNodeDumper.DumpPV(ceresResults.Manager.Context.StartPosAndPriorMoves, ceresResults.BestMoveRoot, true, null);
}
public SuiteTestResult Run(int numConcurrentSuiteThreads = 1, bool outputDetail = true, bool saveCacheWhenDone = true)
{
// Tree reuse is no help, indicate that we won't need it
Def.Engine1Def.SearchParams.TreeReuseEnabled = false;
if (Def.Engine2Def != null)
{
Def.Engine2Def.SearchParams.TreeReuseEnabled = false;
}
// Disable dump for now, the execution parameters are modified
// for the warmup which is confusing because different parameters
// will be chosen for the actual search.
//DumpParams(Def.Output, true);
// Create evaluators
evaluatorSet1 = new NNEvaluatorSet(Def.Engine1Def.EvaluatorDef);
if (Def.Engine2Def != null)
{
evaluatorSet2 = new NNEvaluatorSet(Def.Engine2Def.EvaluatorDef);
}
numConcurrentSuiteThreads = numConcurrentSuiteThreads;
int timerFiredCount = 0;
// TODO: add path automatically
List <EPDEntry> epds = EPDEntry.EPDEntriesInEPDFile(Def.EPDFileName, int.MaxValue);
if (Def.MaxNumPositions == 0)
{
Def.MaxNumPositions = epds.Count;
}
Def.Output.WriteLine();
Def.Output.WriteLine("C1 = " + Def.Engine1Def.EvaluatorDef);
if (Def.RunCeres2Engine)
{
Def.Output.WriteLine("C2 = " + Def.Engine2Def.EvaluatorDef);
}
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine("EX = " + Def.ExternalEngineDef.EngineDef);
}
#if NOT
// To make up for the fact that LZ0 "cheats" by sometimes running over specified number of nodes
// (she seems to always fill the batch even if reached limit), add half a batch extra for Ceres as compensation
if (searchLimitCeres1.Type == SearchLimit.LimitType.NodesPerMove)
{
searchLimitCeres1 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch1.BATCH_SIZE_PRIMARY / 2);
searchLimitCeres2 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch2.BATCH_SIZE_PRIMARY / 2);
}
#endif
//Def.Output.WriteLine($"MAX_CERES_GAME_THREADS {numConcurrentCeresGames} MAX_LEELA_GAME_THREADS {MAX_LEELA_GAME_THREADS}");
// Turn of position reuse if evaluators produce different results
if (Def.RunCeres2Engine && !Def.Engine1Def.EvaluatorDef.NetEvaluationsIdentical(Def.Engine2Def.EvaluatorDef))
{
Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
}
if (Def.RunCeres2Engine && (Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree ||
Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree))
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("\r\nWARNING: REUSE_POSITION_EVALUATIONS_FROM_OTHER_TREE is turned on for one or both evaluators\r\n"
+ "(alternating between the two evaluators). This may cause slight differences in search behavior and speed.\r\n");
Console.ForegroundColor = ConsoleColor.White;
}
Def.Output.WriteLine();
if (Def.MaxNumPositions > epds.Count)
{
Def.MaxNumPositions = epds.Count;
}
epds = epds.GetRange(Def.FirstTestPosition, Def.MaxNumPositions);
int numExternalGameProcesses = 1;
numConcurrentSuiteThreads = Math.Min(Def.MaxNumPositions, numConcurrentSuiteThreads);
if (numConcurrentSuiteThreads > 1)
{
bool evaluator1NonPooled = Def.Engine1Def.EvaluatorDef != null && Def.Engine1Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;
bool evaluator2NonPooled = Def.Engine2Def.EvaluatorDef != null && Def.Engine2Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;
if (evaluator1NonPooled || evaluator2NonPooled)
{
throw new Exception("Must use POOLED neural network evaluator when running suites with parallelism");;
}
if (Def.ExternalEngineDef != null)
{
// For safety (to not overflow main or GPU memory) we limit number of LC0 processes.
const int MAX_LC0_PROCESSES = 4;
numExternalGameProcesses = Math.Min(MAX_LC0_PROCESSES, numConcurrentSuiteThreads);
}
}
bool leelaVerboseMovesStats = true;//xxx Def.NumTestPos == 1;
Func <object> makeExternalEngine = null;
if (Def.ExternalEngineDef != null)
{
if (Def.ExternalEngineDef.EngineDef is GameEngineDefLC0)
{
bool forceDisableSmartPruning = (Def.ExternalEngineDef.EngineDef as GameEngineDefLC0).ForceDisableSmartPruning;
makeExternalEngine = () =>
{
LC0Engine engine = LC0EngineConfigured.GetLC0Engine(null, null, Def.Engine1Def.EvaluatorDef,
NNWeightsFiles.LookupNetworkFile(Def.Engine1Def.EvaluatorDef.Nets[0].Net.NetworkID),
true,
false, leelaVerboseMovesStats, forceDisableSmartPruning);
// WARMUP
engine.AnalyzePositionFromFEN(Position.StartPosition.FEN, null, SearchLimit.NodesPerMove(1));
return(engine);
};
}
else
{
bool resetMovesBetweenMoves = !Def.Engine2Def.SearchParams.TreeReuseEnabled;
bool enableTranpsositions = Def.Engine2Def.SearchParams.Execution.TranspositionMode != TranspositionMode.None;
bool enableTablebases = Def.Engine2Def.SearchParams.EnableTablebases;
makeExternalEngine = () => Def.ExternalEngineDef.EngineDef.CreateEngine();
}
}
// Don't create too many non_Ceres threads since each one will consume seaprate GPU memory or threads
int maxLeelaThreads = Math.Min(numExternalGameProcesses, numConcurrentSuiteThreads);
ObjectPool <object> externalEnginePool = new ObjectPool <object>(makeExternalEngine, maxLeelaThreads);
using (new TimingBlock("EPDS"))
{
Parallel.For(0, epds.Count,
new ParallelOptions()
{
MaxDegreeOfParallelism = numConcurrentSuiteThreads
},
delegate(int gameNum)
{
try
{
EPDEntry epd = epds[gameNum];
// Skip positions which are already draws
if (epd.Position.CheckDrawBasedOnMaterial == Position.PositionDrawStatus.DrawByInsufficientMaterial)
{
return;
}
// TODO: also do this for checkmate?
ProcessEPD(gameNum, epds[gameNum], outputDetail, externalEnginePool);
}
catch (Exception exc)
{
Def.Output.WriteLine("Error in ProcessEPD " + exc);
throw exc;
}
});
}
Def.Output.WriteLine();
Def.Output.WriteLine();
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine($"Total {Def.ExternalEngineDef.ID} Time {totalTimeOther,6:F2}");
}
Def.Output.WriteLine($"Total C1 Time {totalTimeCeres1,6:F2}");
if (Def.CeresEngine2Def != null)
{
Def.Output.WriteLine($"Total C2 Time {totalTimeCeres2,6:F2}");
}
Def.Output.WriteLine();
if (Def.ExternalEngineDef != null)
{
Def.Output.WriteLine($"Avg {Def.ExternalEngineDef.ID} pos/sec {totalNodesOther / totalTimeOther,8:F2}");
}
Def.Output.WriteLine($"Avg Ceres pos/sec {totalNodes1 / totalTimeCeres1,8:F2}");
if (Def.CeresEngine2Def != null)
{
Def.Output.WriteLine($"Avg Ceres2 pos/sec {totalNodes2 / totalTimeCeres2,8:F2}");
}
Def.Output.WriteLine();
Def.Output.WriteLine();
EngineCeres1.Dispose();
EngineCeres2?.Dispose();
EngineExternal?.Dispose();
externalEnginePool.Shutdown(engineObj => (engineObj as LC0Engine).Dispose());
evaluatorSet1.Dispose();
evaluatorSet2?.Dispose();
return(new SuiteTestResult(Def)
{
AvgScore1 = (float)accCeres1 / numSearches,
AvgScore2 = (float)accCeres2 / numSearches,
AvgWScore1 = (float)accWCeres1 / numSearches,
AvgWScore2 = (float)accWCeres2 / numSearches,
AvgScoreLC0 = (float)avgOther / numSearches,
TotalRuntimeLC0 = totalTimeOther,
TotalRuntime1 = totalTimeCeres1,
TotalRuntime2 = totalTimeCeres2,
TotalNodesLC0 = totalNodesOther,
TotalNodes1 = totalNodes1,
TotalNodes2 = totalNodes2
});
}