Example #1
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="store"></param>
        /// <param name="reuseOtherContextForEvaluatedNodes"></param>
        /// <param name="reusePositionCache"></param>
        /// <param name="reuseTranspositionRoots"></param>
        /// <param name="nnEvaluators"></param>
        /// <param name="searchParams"></param>
        /// <param name="childSelectParams"></param>
        /// <param name="searchLimit"></param>
        /// <param name="paramsSearchExecutionPostprocessor"></param>
        /// <param name="limitManager"></param>
        /// <param name="startTime"></param>
        /// <param name="priorManager"></param>
        /// <param name="gameMoveHistory"></param>
        /// <param name="isFirstMoveOfGame"></param>
        public MCTSManager(MCTSNodeStore store,
                           MCTSIterator reuseOtherContextForEvaluatedNodes,
                           PositionEvalCache reusePositionCache,
                           TranspositionRootsDict reuseTranspositionRoots,
                           NNEvaluatorSet nnEvaluators,
                           ParamsSearch searchParams,
                           ParamsSelect childSelectParams,
                           SearchLimit searchLimit,
                           ParamsSearchExecutionModifier paramsSearchExecutionPostprocessor,
                           IManagerGameLimit limitManager,
                           DateTime startTime,
                           MCTSManager priorManager,
                           List <GameMoveStat> gameMoveHistory,
                           bool isFirstMoveOfGame)
        {
            StartTimeThisSearch                = startTime;
            RootNWhenSearchStarted             = store.Nodes.nodes[store.RootIndex.Index].N;
            ParamsSearchExecutionPostprocessor = paramsSearchExecutionPostprocessor;
            IsFirstMoveOfGame = isFirstMoveOfGame;

            PriorMoveStats = new List <GameMoveStat>();

            // Make our own copy of move history.
            if (gameMoveHistory != null)
            {
                PriorMoveStats.AddRange(gameMoveHistory);
            }

            // Possibly convert time limit per game into time for this move.
            if (searchLimit.IsPerGameLimit)
            {
                SearchLimitType type = searchLimit.Type == SearchLimitType.SecondsForAllMoves
                                                       ? SearchLimitType.SecondsPerMove
                                                       : SearchLimitType.NodesPerMove;
                float rootQ = priorManager == null ? float.NaN : (float)store.RootNode.Q;


                limitsManagerInputs = new(store.Nodes.PriorMoves.FinalPosition,
                                          searchParams, PriorMoveStats,
                                          type, store.RootNode.N, rootQ,
                                          searchLimit.Value, searchLimit.ValueIncrement,
                                          float.NaN, float.NaN,
                                          maxMovesToGo : searchLimit.MaxMovesToGo,
                                          isFirstMoveOfGame : isFirstMoveOfGame);

                ManagerGameLimitOutputs timeManagerOutputs = limitManager.ComputeMoveAllocation(limitsManagerInputs);
                SearchLimit = timeManagerOutputs.LimitTarget;
            }
            else
            {
                SearchLimit = searchLimit;
            }

            // Possibly autoselect new optimal parameters
            ParamsSearchExecutionChooser paramsChooser = new ParamsSearchExecutionChooser(nnEvaluators.EvaluatorDef,
                                                                                          searchParams, childSelectParams, searchLimit);

            // TODO: technically this is overwriting the params belonging to the prior search, that's ugly (but won't actually cause a problem)
            paramsChooser.ChooseOptimal(searchLimit.EstNumNodes(50_000, false), paramsSearchExecutionPostprocessor); // TODO: make 50_000 smarter


            int estNumNodes = EstimatedNumSearchNodesForEvaluator(searchLimit, nnEvaluators);

            // Adjust the nodes estimate if we are continuing an existing search
            if (searchLimit.Type == SearchLimitType.NodesPerMove && RootNWhenSearchStarted > 0)
            {
                estNumNodes = Math.Max(0, estNumNodes - RootNWhenSearchStarted);
            }
            Context = new MCTSIterator(store, reuseOtherContextForEvaluatedNodes, reusePositionCache, reuseTranspositionRoots,
                                       nnEvaluators, searchParams, childSelectParams, searchLimit, estNumNodes);
            ThreadSearchContext = Context;

            TerminationManager = new MCTSFutilityPruning(this, Context);
            LimitManager       = limitManager;

            CeresEnvironment.LogInfo("MCTS", "Init", $"SearchManager created for store {store}", InstanceID);
        }
Example #2
0
        public static void Analyze(string fen, SearchLimit searchLimit,
                                   NNEvaluatorDef evaluatorDef,
                                   bool forceDisablePruning,
                                   LC0Engine lc0Engine         = null,
                                   GameEngine comparisonEngine = null,
                                   bool verbose = false)
        {
            Console.WriteLine("=============================================================================");
            Console.WriteLine("Analyzing FEN   : " + fen);
            Console.WriteLine("Search limit    : " + searchLimit.ToString());
            Console.WriteLine("Ceres evaluator : " + evaluatorDef.ToString());
            if (comparisonEngine != null)
            {
                Console.WriteLine("Opponent      : " + comparisonEngine.ToString());
            }
            Console.WriteLine();
            Console.WriteLine();

            NNEvaluatorSet nnEvaluators = new NNEvaluatorSet(evaluatorDef);

            // Warmup (in parallel)
            lc0Engine?.DoSearchPrepare();
            Parallel.Invoke(
                () => nnEvaluators.Warmup(true),
                () => comparisonEngine?.Warmup());

            bool ceresDone = false;

            lastInfoUpdate = DateTime.Now;

            UCISearchInfo lastCeresInfo = null;

            // Launch Ceres
            MCTSearch ceresResults = null;
            Task      searchCeres  = Task.Run(() =>
            {
                ParamsSearch searchParams = new ParamsSearch();
                searchParams.FutilityPruningStopSearchEnabled = !forceDisablePruning;
                PositionWithHistory positionWithHistory       = PositionWithHistory.FromFENAndMovesUCI(fen, null);
                ceresResults = new MCTSearch();
                ceresResults.Search(nnEvaluators, new ParamsSelect(), searchParams, null, null,
                                    null, positionWithHistory, searchLimit, verbose, DateTime.Now, null,
                                    manager => lastCeresInfo = new UCISearchInfo(UCIManager.UCIInfoString(manager), null, null), false, true);
            });

            // Possibly launch search for other engine
            Task searchComparison = null;

            if (lc0Engine != null || comparisonEngine != null)
            {
                searchComparison = Task.Run(() =>
                {
                    if (lc0Engine != null)
                    {
                        lc0Engine.DoSearchPrepare();
                        lc0Engine.AnalyzePositionFromFENAndMoves(fen, null, fen, searchLimit);
                    }
                    else
                    {
                        // TODO: someday enable passing in of moves here
                        comparisonEngine.Search(PositionWithHistory.FromFENAndMovesUCI(fen, null), searchLimit, verbose: true);
                    }
                });
            }
            ;

            while (!searchCeres.IsCompleted || (searchComparison != null && !searchComparison.IsCompleted))
            {
                Thread.Sleep(1000);
//Console.WriteLine(DateTime.Now + " --> " + lastCeresInfo?.PVString + " OTHER " + comparisonEngine?.UCIInfo?.RawString);

                int numCharactersSame = int.MaxValue;
                if (lastCeresInfo?.PVString != null || comparisonEngine?.UCIInfo?.RawString != null)
                {
                    if (lastCeresInfo != null && comparisonEngine?.UCIInfo != null)
                    {
                        numCharactersSame = 0;
                        string        pv1 = lastCeresInfo.PVString;
                        UCISearchInfo lastComparisonInfo = comparisonEngine.UCIInfo;
                        string        pv2 = lastComparisonInfo.PVString;
                        while (pv1.Length > numCharactersSame &&
                               pv2.Length > numCharactersSame &&
                               pv1[numCharactersSame] == pv2[numCharactersSame])
                        {
                            numCharactersSame++;
                        }
                    }
                }

                if (lastCeresInfo != null)
                {
                    WriteUCI("Ceres", lastCeresInfo, numCharactersSame);
                }

                if (comparisonEngine != null)
                {
                    WriteUCI(comparisonEngine.ID, comparisonEngine.UCIInfo, numCharactersSame);
                }
                Console.WriteLine();
            }

            searchCeres.Wait();
            searchComparison?.Wait();

            string infoUpdate = UCIManager.UCIInfoString(ceresResults.Manager);

            double q2 = ceresResults.BestMoveRoot.Q;

            //SearchPrincipalVariation pv2 = new SearchPrincipalVariation(worker2.Root);
            MCTSPosTreeNodeDumper.DumpPV(ceresResults.Manager.Context.StartPosAndPriorMoves, ceresResults.BestMoveRoot, true, null);
        }
Example #3
0
        public SuiteTestResult Run(int numConcurrentSuiteThreads = 1, bool outputDetail = true, bool saveCacheWhenDone = true)
        {
            // Tree reuse is no help, indicate that we won't need it
            Def.Engine1Def.SearchParams.TreeReuseEnabled = false;
            if (Def.Engine2Def != null)
            {
                Def.Engine2Def.SearchParams.TreeReuseEnabled = false;
            }

            // Disable dump for now, the execution parameters are modified
            // for the warmup which is confusing because different parameters
            // will be chosen for the actual search.
            //DumpParams(Def.Output, true);

            // Create evaluators
            evaluatorSet1 = new NNEvaluatorSet(Def.Engine1Def.EvaluatorDef);
            if (Def.Engine2Def != null)
            {
                evaluatorSet2 = new NNEvaluatorSet(Def.Engine2Def.EvaluatorDef);
            }

            numConcurrentSuiteThreads = numConcurrentSuiteThreads;

            int timerFiredCount = 0;

            // TODO: add path automatically
            List <EPDEntry> epds = EPDEntry.EPDEntriesInEPDFile(Def.EPDFileName, int.MaxValue);

            if (Def.MaxNumPositions == 0)
            {
                Def.MaxNumPositions = epds.Count;
            }

            Def.Output.WriteLine();
            Def.Output.WriteLine("C1 = " + Def.Engine1Def.EvaluatorDef);
            if (Def.RunCeres2Engine)
            {
                Def.Output.WriteLine("C2 = " + Def.Engine2Def.EvaluatorDef);
            }
            if (Def.ExternalEngineDef != null)
            {
                Def.Output.WriteLine("EX = " + Def.ExternalEngineDef.EngineDef);
            }

#if NOT
            // To make up for the fact that LZ0 "cheats" by sometimes running over specified number of nodes
            // (she seems to always fill the batch even if reached limit), add half a batch extra for Ceres as compensation
            if (searchLimitCeres1.Type == SearchLimit.LimitType.NodesPerMove)
            {
                searchLimitCeres1 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch1.BATCH_SIZE_PRIMARY / 2);
                searchLimitCeres2 = new SearchLimit(searchLimit.Type, searchLimit.Value + paramsSearch2.BATCH_SIZE_PRIMARY / 2);
            }
#endif

            //Def.Output.WriteLine($"MAX_CERES_GAME_THREADS {numConcurrentCeresGames} MAX_LEELA_GAME_THREADS {MAX_LEELA_GAME_THREADS}");

            // Turn of position reuse if evaluators produce different results
            if (Def.RunCeres2Engine && !Def.Engine1Def.EvaluatorDef.NetEvaluationsIdentical(Def.Engine2Def.EvaluatorDef))
            {
                Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
                Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree = false;
            }

            if (Def.RunCeres2Engine && (Def.Engine1Def.SearchParams.ReusePositionEvaluationsFromOtherTree ||
                                        Def.Engine2Def.SearchParams.ReusePositionEvaluationsFromOtherTree))
            {
                Console.ForegroundColor = ConsoleColor.Cyan;
                Console.WriteLine("\r\nWARNING: REUSE_POSITION_EVALUATIONS_FROM_OTHER_TREE is turned on for one or both evaluators\r\n"
                                  + "(alternating between the two evaluators). This may cause slight differences in search behavior and speed.\r\n");
                Console.ForegroundColor = ConsoleColor.White;
            }

            Def.Output.WriteLine();

            if (Def.MaxNumPositions > epds.Count)
            {
                Def.MaxNumPositions = epds.Count;
            }
            epds = epds.GetRange(Def.FirstTestPosition, Def.MaxNumPositions);

            int numExternalGameProcesses = 1;

            numConcurrentSuiteThreads = Math.Min(Def.MaxNumPositions, numConcurrentSuiteThreads);

            if (numConcurrentSuiteThreads > 1)
            {
                bool evaluator1NonPooled = Def.Engine1Def.EvaluatorDef != null && Def.Engine1Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;
                bool evaluator2NonPooled = Def.Engine2Def.EvaluatorDef != null && Def.Engine2Def.EvaluatorDef.DeviceCombo != Chess.NNEvaluators.Defs.NNEvaluatorDeviceComboType.Pooled;

                if (evaluator1NonPooled || evaluator2NonPooled)
                {
                    throw new Exception("Must use POOLED neural network evaluator when running suites with parallelism");;
                }

                if (Def.ExternalEngineDef != null)
                {
                    // For safety (to not overflow main or GPU memory) we limit number of LC0 processes.
                    const int MAX_LC0_PROCESSES = 4;
                    numExternalGameProcesses = Math.Min(MAX_LC0_PROCESSES, numConcurrentSuiteThreads);
                }
            }

            bool          leelaVerboseMovesStats = true;//xxx Def.NumTestPos == 1;
            Func <object> makeExternalEngine     = null;

            if (Def.ExternalEngineDef != null)
            {
                if (Def.ExternalEngineDef.EngineDef is GameEngineDefLC0)
                {
                    bool forceDisableSmartPruning = (Def.ExternalEngineDef.EngineDef as GameEngineDefLC0).ForceDisableSmartPruning;
                    makeExternalEngine = () =>
                    {
                        LC0Engine engine = LC0EngineConfigured.GetLC0Engine(null, null, Def.Engine1Def.EvaluatorDef,
                                                                            NNWeightsFiles.LookupNetworkFile(Def.Engine1Def.EvaluatorDef.Nets[0].Net.NetworkID),
                                                                            true,
                                                                            false, leelaVerboseMovesStats, forceDisableSmartPruning);
                        // WARMUP
                        engine.AnalyzePositionFromFEN(Position.StartPosition.FEN, null, SearchLimit.NodesPerMove(1));
                        return(engine);
                    };
                }
                else
                {
                    bool resetMovesBetweenMoves = !Def.Engine2Def.SearchParams.TreeReuseEnabled;
                    bool enableTranpsositions   = Def.Engine2Def.SearchParams.Execution.TranspositionMode != TranspositionMode.None;
                    bool enableTablebases       = Def.Engine2Def.SearchParams.EnableTablebases;

                    makeExternalEngine = () => Def.ExternalEngineDef.EngineDef.CreateEngine();
                }
            }

            // Don't create too many non_Ceres threads since each one will consume seaprate GPU memory or threads
            int maxLeelaThreads = Math.Min(numExternalGameProcesses, numConcurrentSuiteThreads);
            ObjectPool <object> externalEnginePool = new ObjectPool <object>(makeExternalEngine, maxLeelaThreads);

            using (new TimingBlock("EPDS"))
            {
                Parallel.For(0, epds.Count,
                             new ParallelOptions()
                {
                    MaxDegreeOfParallelism = numConcurrentSuiteThreads
                },
                             delegate(int gameNum)
                {
                    try
                    {
                        EPDEntry epd = epds[gameNum];

                        // Skip positions which are already draws
                        if (epd.Position.CheckDrawBasedOnMaterial == Position.PositionDrawStatus.DrawByInsufficientMaterial)
                        {
                            return;
                        }
                        // TODO: also do this for checkmate?

                        ProcessEPD(gameNum, epds[gameNum], outputDetail, externalEnginePool);
                    }
                    catch (Exception exc)
                    {
                        Def.Output.WriteLine("Error in ProcessEPD " + exc);
                        throw exc;
                    }
                });
            }

            Def.Output.WriteLine();

            Def.Output.WriteLine();
            if (Def.ExternalEngineDef != null)
            {
                Def.Output.WriteLine($"Total {Def.ExternalEngineDef.ID} Time {totalTimeOther,6:F2}");
            }
            Def.Output.WriteLine($"Total C1 Time {totalTimeCeres1,6:F2}");
            if (Def.CeresEngine2Def != null)
            {
                Def.Output.WriteLine($"Total C2 Time {totalTimeCeres2,6:F2}");
            }

            Def.Output.WriteLine();
            if (Def.ExternalEngineDef != null)
            {
                Def.Output.WriteLine($"Avg {Def.ExternalEngineDef.ID} pos/sec    {totalNodesOther / totalTimeOther,8:F2}");
            }
            Def.Output.WriteLine($"Avg Ceres    pos/sec    {totalNodes1 / totalTimeCeres1,8:F2}");
            if (Def.CeresEngine2Def != null)
            {
                Def.Output.WriteLine($"Avg Ceres2    pos/sec    {totalNodes2 / totalTimeCeres2,8:F2}");
            }

            Def.Output.WriteLine();
            Def.Output.WriteLine();


            EngineCeres1.Dispose();
            EngineCeres2?.Dispose();
            EngineExternal?.Dispose();

            externalEnginePool.Shutdown(engineObj => (engineObj as LC0Engine).Dispose());
            evaluatorSet1.Dispose();
            evaluatorSet2?.Dispose();

            return(new SuiteTestResult(Def)
            {
                AvgScore1 = (float)accCeres1 / numSearches,
                AvgScore2 = (float)accCeres2 / numSearches,
                AvgWScore1 = (float)accWCeres1 / numSearches,
                AvgWScore2 = (float)accWCeres2 / numSearches,
                AvgScoreLC0 = (float)avgOther / numSearches,

                TotalRuntimeLC0 = totalTimeOther,
                TotalRuntime1 = totalTimeCeres1,
                TotalRuntime2 = totalTimeCeres2,

                TotalNodesLC0 = totalNodesOther,
                TotalNodes1 = totalNodes1,
                TotalNodes2 = totalNodes2
            });
        }