Esempio n. 1
0
        private Move GetBestMove(IState rootState)
        {
            var rootNode = new MonteCarloTreeSearchNode(rootState, GetMoves);

            while (!cancel.Cancelled)
            {
                var node  = rootNode;
                var state = rootState.Clone();

                simulationCount++;

                // Select
                while (node.Untried.Count == 0 && node.Children.Count > 0)
                {
                    node = node.SelectChild();
                    state.MakeMove(node.Move);
                    visitedNodes++;
                }

                // Expand
                if (node.Untried.Count > 0)
                {
                    var move = node.Untried[random.Next(node.Untried.Count)];
                    state.MakeMove(move);
                    node = node.AddChild(state, move);
                    visitedNodes++;
                }

                // Simulate
                while (state.GetWinner() == Player.None)
                {
                    var moves = state.GetAllMoves();
                    if (moves.Count == 0)
                    {
                        break;
                    }
                    state.MakeMove(moves[random.Next(moves.Count)]);
                    visitedNodes++;
                }

                // Backpropagate
                while (node != null)
                {
                    var winner = state.GetWinner();
                    node.Update(winner == node.LastToMove ? 1.0 : (winner == Player.None ? 0.5 : 0.0));
                    node = node.Parent;
                    visitedNodes++;
                }
            }

            if (debugMode)
            {
                var builder = new StringBuilder();
                DebugTree(builder, rootNode, 0, 5);
                debugStatus = builder.ToString();
            }

            return(rootNode.Children.OrderBy(x => x.Visits).LastOrDefault().Move);
        }
            public MonteCarloTreeSearchNode AddChild(Board state, int move)
            {
                var newNode = new MonteCarloTreeSearchNode(state, move, this, _getMoves);

                Untried.Remove(move);
                Children.Add(newNode);
                return(newNode);
            }
Esempio n. 3
0
        private void DebugTree(StringBuilder sb, MonteCarloTreeSearchNode node, int indent, int max)
        {
            foreach (var c in node.Children.OrderByDescending(x => x.Visits).ThenByDescending(x => x.Wins))
            {
                sb.Append(new string(' ', indent * 4));
                sb.AppendLine(c.Move?.X + "," + c.Move?.Y + " (" + c.Wins + "/" + c.Visits + ")");

                if (indent < max)
                {
                    DebugTree(sb, c, indent + 1, max);
                }
            }
        }
            public MonteCarloTreeSearchNode(Board state, int move, MonteCarloTreeSearchNode parent, Func <Board, List <int> > getMoves)
            {
                _getMoves = getMoves;

                Move   = move;
                Parent = parent;

                Children = new List <MonteCarloTreeSearchNode>();
                Wins     = 0.0;
                Visits   = 0.0;

                if (state != null)
                {
                    Untried    = _getMoves(state);
                    LastToMove = state.GetPlayerForPreviousTurn();
                }
                else
                {
                    Untried = new List <int>();
                }
            }
Esempio n. 5
0
            public MonteCarloTreeSearchNode(IState state, Move move, MonteCarloTreeSearchNode parent, Func <IState, List <Move> > getMoves)
            {
                _getMoves = getMoves;

                Move   = move;
                Parent = parent;

                Children = new List <MonteCarloTreeSearchNode>();
                Wins     = 0.0;
                Visits   = 0.0;

                if (state != null)
                {
                    Untried    = _getMoves(state);
                    LastToMove = state.PreviousPlayerToMove;
                }
                else
                {
                    Untried = new List <Move>();
                }
            }
 private void ReportStatus(MonteCarloTreeSearchNode rootNode)
 {
     // Update Status
     if (statusUpdate.ElapsedMilliseconds > millisecondsBetweenUpdates && OnStatus != null)
     {
         EngineStatus status = new EngineStatus();
         foreach (var child in rootNode.Children)
         {
             double eval = Math.Round((child.Visits > 0 ? 200.0 * child.Wins / child.Visits : 0) - 100.0, 2);
             string pv   = "";
             var    c    = child;
             while (c != null && c.Move >= 0 && c.Move <= 80)
             {
                 pv += /*(c.LastToMove == Player.One ? "b " : "o ") +*/ Constants.TileNames[c.Move] + " (" + c.Wins + "/" + c.Visits + ")   ";
                 c   = c.Children?.OrderBy(x => x.Visits)?.ThenBy(x => x.Wins)?.LastOrDefault();
             }
             status.Add(child?.Move ?? 80, eval, pv, child.Visits);
         }
         status.Sort();
         OnStatus?.Invoke(this, status);
         statusUpdate = Stopwatch.StartNew();
     }
 }
        private int MonteCarloTreeSearch(Board rootState)
        {
            var rootNode = new MonteCarloTreeSearchNode(rootState, GetMoves);
            var forceEnd = false;

            var parallelism = 8;
            var semaphore   = new Semaphore(parallelism, parallelism);

            for (int i = 0; i < maxIterations && !cancel.Cancelled && !forceEnd; i++)
            {
                var node  = rootNode;
                var state = new Board(rootState);
                simulationCount++;

                // Select
                while (node.Untried.Count == 0 && node.Children.Count > 0)
                {
                    node = node.SelectChild();
                    state.MakeMove(node.Move);
                    visitedNodes++;
                }

                // Expand
                if (node.Untried.Count > 0)
                {
                    var move = node.Untried[random.Next(node.Untried.Count)];
                    state.MakeMove(move);
                    node = node.AddChild(state, move);
                    visitedNodes++;
                }

                semaphore.WaitOne();

                Task.Run(() =>
                {
                    // Simulate
                    while (state.Winner == Player.Empty && state.Turn < VolcanoGame.Settings.TournamentAdjudicateMaxTurns)
                    {
                        var moves = GetRandomMoves(state);
                        if (moves.Count == 0)
                        {
                            break;
                        }
                        state.MakeMove(moves[random.Next(moves.Count)]);
                        visitedNodes++;
                    }

                    // Backpropagate
                    while (node != null)
                    {
                        if (forceEnd)
                        {
                            break;
                        }

                        //var fastWinReward = 0.5 * (state.Turn - rootState.Turn) / VolcanoGame.Settings.TournamentAdjudicateMaxTurns;
                        //node.Update(state.Winner == node.LastToMove ? 1.0 - fastWinReward : 0.0);

                        node.Update(state.Winner == node.LastToMove ? 1.0 : 0.0);
                        node = node.Parent;
                        visitedNodes++;
                    }

                    semaphore.Release();
                });

                // Cut Short
                foreach (var child in rootNode.Children)
                {
                    // If we have a potential move that has a 100% win rate and it's been visited a lot of times, stop searching
                    if (child.Visits > 500 && child.Wins == child.Visits)
                    {
                        forceEnd = true;
                    }
                }

                // Update Status
                if (statusUpdate.ElapsedMilliseconds > millisecondsBetweenUpdates && OnStatus != null)
                {
                    EngineStatus status = new EngineStatus();
                    foreach (var child in rootNode.Children)
                    {
                        double eval = Math.Round((child.Visits > 0 ? 200.0 * Math.Min(child.Wins, child.Visits) / child.Visits : 0) - 100.0, 2);
                        string pv   = "";
                        var    c    = child;
                        while (c != null && c.Move >= 0 && c.Move <= 80)
                        {
                            pv += Constants.TileNames[c.Move] + " (" + Math.Round(c.Wins, 2) + "/" + c.Visits + ")   ";
                            c   = c.Children?.OrderBy(x => x.Visits)?.ThenBy(x => x.Wins)?.LastOrDefault();
                        }
                        status.Add(child?.Move ?? 80, eval, pv, child.Visits);
                    }
                    status.Sort();
                    OnStatus(this, status);
                    statusUpdate = Stopwatch.StartNew();
                }
            }

            forceEnd = true;

            return(rootNode.Children.OrderBy(x => x.Visits).LastOrDefault().Move);
        }
 private double UpperConfidenceBound(MonteCarloTreeSearchNode node)
 {
     return(node.Wins / node.Visits + Math.Sqrt(2.0 * Math.Log(Visits) / node.Visits));
 }
        private int MonteCarloTreeSearch(Board rootState)
        {
            var rootNode = new MonteCarloTreeSearchNode(rootState, GetMoves);
            var forceWin = false;

            while (!cancel.Cancelled && !forceWin)
            {
                var node  = rootNode;
                var state = new Board(rootState);

                state.fastWinSearch = _allowFastWinSearch;

                simulationCount++;

                // Select
                while (node.Untried.Count == 0 && node.Children.Count > 0)
                {
                    node = node.SelectChild(_ucbFactor);
                    state.MakeMove(node.Move);
                    visitedNodes++;
                }

                // Expand
                if (node.Untried.Count > 0)
                {
                    var move = node.Untried[random.Next(node.Untried.Count)];
                    state.MakeMove(move);
                    node = node.AddChild(state, move);
                    visitedNodes++;
                }

                // Simulate
                while (state.Winner == Player.Empty && state.Turn < VolcanoGame.Settings.TournamentAdjudicateMaxTurns)
                {
                    var moves = state.GetMoves();
                    if (moves.Count == 0)
                    {
                        break;
                    }
                    state.MakeMove(moves[random.Next(moves.Count)]);
                    visitedNodes++;
                }

                // Backpropagate
                while (node != null)
                {
                    node.Update(state.Winner == node.LastToMove ? 1.0 : 0.0);
                    node = node.Parent;
                    visitedNodes++;
                }

                // Cut Short
                if (_allowForcedWins)
                {
                    foreach (var child in rootNode.Children)
                    {
                        // If we have a potential move that has a 100% win rate and it's been visited a lot of times, stop searching
                        if (child.Visits > 500 && child.Wins == child.Visits)
                        {
                            forceWin = true;
                        }
                    }
                }

                ReportStatus(rootNode);
            }

            ReportStatus(rootNode);

            return(rootNode.Children.OrderBy(x => x.Visits).LastOrDefault().Move);
        }
Esempio n. 10
0
        private int MonteCarloTreeSearch(Board rootState)
        {
            var rootNode = new MonteCarloTreeSearchNode(rootState, GetMoves);
            var forceWin = false;

            while (!cancel.Cancelled && !forceWin)
            {
                var node  = rootNode;
                var state = new Board(rootState);

                state.allowHash     = _allowHash;
                state.fastWinSearch = _allowFastWinSearch;

                if (_allowHash)
                {
                    state.winHashes = winHashes;
                }

                simulationCount++;

                // Select
                while (node.Untried.Count == 0 && node.Children.Count > 0)
                {
                    node = node.SelectChild(_ucbFactor);
                    state.MakeMove(node.Move);
                    visitedNodes++;
                }

                // Expand
                if (node.Untried.Count > 0)
                {
                    var move = node.Untried[random.Next(node.Untried.Count)];
                    state.MakeMove(move);
                    node = node.AddChild(state, move);
                    visitedNodes++;
                }

                // Simulate
                while (state.Winner == Player.Empty && state.Turn < VolcanoGame.Settings.TournamentAdjudicateMaxTurns)
                {
                    var moves = state.GetMoves();
                    if (moves.Count == 0)
                    {
                        break;
                    }
                    state.MakeMove(moves[random.Next(moves.Count)]);
                    visitedNodes++;
                }

                // Backpropagate
                while (node != null)
                {
                    node.Update(state.Winner == node.LastToMove ? 1.0 : 0.0);
                    node = node.Parent;
                    visitedNodes++;
                }

                // Cut Short
                if (_allowForcedWins)
                {
                    foreach (var child in rootNode.Children)
                    {
                        // If we have a potential move that has a 100% win rate and it's been visited a lot of times, stop searching
                        if (child.Visits > 500 && child.Wins == child.Visits)
                        {
                            forceWin = true;
                        }
                    }
                }

                // Update Status
                if (statusUpdate.ElapsedMilliseconds > millisecondsBetweenUpdates && OnStatus != null)
                {
                    EngineStatus status = new EngineStatus();
                    foreach (var child in rootNode.Children)
                    {
                        double eval = Math.Round((child.Visits > 0 ? 200.0 * child.Wins / child.Visits : 0) - 100.0, 2);
                        string pv   = "";
                        var    c    = child;
                        while (c != null && c.Move >= 0 && c.Move <= 80)
                        {
                            pv += Constants.TileNames[c.Move] + " (" + c.Wins + "/" + c.Visits + ")   ";
                            c   = c.Children?.OrderBy(x => x.Visits)?.ThenBy(x => x.Wins)?.LastOrDefault();
                        }
                        status.Add(child?.Move ?? 80, eval, pv, child.Visits);
                    }
                    status.Sort();
                    OnStatus?.Invoke(this, status);
                    statusUpdate = Stopwatch.StartNew();
                }
            }

            return(rootNode.Children.OrderBy(x => x.Visits).LastOrDefault().Move);
        }