private Move GetBestMove(IState rootState)
{
var rootNode = new MonteCarloTreeSearchNode(rootState, GetMoves);
while (!cancel.Cancelled)
{
var node = rootNode;
var state = rootState.Clone();
simulationCount++;
// Select
while (node.Untried.Count == 0 && node.Children.Count > 0)
{
node = node.SelectChild();
state.MakeMove(node.Move);
visitedNodes++;
}
// Expand
if (node.Untried.Count > 0)
{
var move = node.Untried[random.Next(node.Untried.Count)];
state.MakeMove(move);
node = node.AddChild(state, move);
visitedNodes++;
}
// Simulate
while (state.GetWinner() == Player.None)
{
var moves = state.GetAllMoves();
if (moves.Count == 0)
{
break;
}
state.MakeMove(moves[random.Next(moves.Count)]);
visitedNodes++;
}
// Backpropagate
while (node != null)
{
var winner = state.GetWinner();
node.Update(winner == node.LastToMove ? 1.0 : (winner == Player.None ? 0.5 : 0.0));
node = node.Parent;
visitedNodes++;
}
}
if (debugMode)
{
var builder = new StringBuilder();
DebugTree(builder, rootNode, 0, 5);
debugStatus = builder.ToString();
}
return(rootNode.Children.OrderBy(x => x.Visits).LastOrDefault().Move);
}
public MonteCarloTreeSearchNode AddChild(Board state, int move)
{
var newNode = new MonteCarloTreeSearchNode(state, move, this, _getMoves);
Untried.Remove(move);
Children.Add(newNode);
return(newNode);
}
private void DebugTree(StringBuilder sb, MonteCarloTreeSearchNode node, int indent, int max)
{
foreach (var c in node.Children.OrderByDescending(x => x.Visits).ThenByDescending(x => x.Wins))
{
sb.Append(new string(' ', indent * 4));
sb.AppendLine(c.Move?.X + "," + c.Move?.Y + " (" + c.Wins + "/" + c.Visits + ")");
if (indent < max)
{
DebugTree(sb, c, indent + 1, max);
}
}
}
public MonteCarloTreeSearchNode(Board state, int move, MonteCarloTreeSearchNode parent, Func <Board, List <int> > getMoves)
{
_getMoves = getMoves;
Move = move;
Parent = parent;
Children = new List <MonteCarloTreeSearchNode>();
Wins = 0.0;
Visits = 0.0;
if (state != null)
{
Untried = _getMoves(state);
LastToMove = state.GetPlayerForPreviousTurn();
}
else
{
Untried = new List <int>();
}
}
public MonteCarloTreeSearchNode(IState state, Move move, MonteCarloTreeSearchNode parent, Func <IState, List <Move> > getMoves)
{
_getMoves = getMoves;
Move = move;
Parent = parent;
Children = new List <MonteCarloTreeSearchNode>();
Wins = 0.0;
Visits = 0.0;
if (state != null)
{
Untried = _getMoves(state);
LastToMove = state.PreviousPlayerToMove;
}
else
{
Untried = new List <Move>();
}
}
private void ReportStatus(MonteCarloTreeSearchNode rootNode)
{
// Update Status
if (statusUpdate.ElapsedMilliseconds > millisecondsBetweenUpdates && OnStatus != null)
{
EngineStatus status = new EngineStatus();
foreach (var child in rootNode.Children)
{
double eval = Math.Round((child.Visits > 0 ? 200.0 * child.Wins / child.Visits : 0) - 100.0, 2);
string pv = "";
var c = child;
while (c != null && c.Move >= 0 && c.Move <= 80)
{
pv += /*(c.LastToMove == Player.One ? "b " : "o ") +*/ Constants.TileNames[c.Move] + " (" + c.Wins + "/" + c.Visits + ") ";
c = c.Children?.OrderBy(x => x.Visits)?.ThenBy(x => x.Wins)?.LastOrDefault();
}
status.Add(child?.Move ?? 80, eval, pv, child.Visits);
}
status.Sort();
OnStatus?.Invoke(this, status);
statusUpdate = Stopwatch.StartNew();
}
}
private int MonteCarloTreeSearch(Board rootState)
{
var rootNode = new MonteCarloTreeSearchNode(rootState, GetMoves);
var forceEnd = false;
var parallelism = 8;
var semaphore = new Semaphore(parallelism, parallelism);
for (int i = 0; i < maxIterations && !cancel.Cancelled && !forceEnd; i++)
{
var node = rootNode;
var state = new Board(rootState);
simulationCount++;
// Select
while (node.Untried.Count == 0 && node.Children.Count > 0)
{
node = node.SelectChild();
state.MakeMove(node.Move);
visitedNodes++;
}
// Expand
if (node.Untried.Count > 0)
{
var move = node.Untried[random.Next(node.Untried.Count)];
state.MakeMove(move);
node = node.AddChild(state, move);
visitedNodes++;
}
semaphore.WaitOne();
Task.Run(() =>
{
// Simulate
while (state.Winner == Player.Empty && state.Turn < VolcanoGame.Settings.TournamentAdjudicateMaxTurns)
{
var moves = GetRandomMoves(state);
if (moves.Count == 0)
{
break;
}
state.MakeMove(moves[random.Next(moves.Count)]);
visitedNodes++;
}
// Backpropagate
while (node != null)
{
if (forceEnd)
{
break;
}
//var fastWinReward = 0.5 * (state.Turn - rootState.Turn) / VolcanoGame.Settings.TournamentAdjudicateMaxTurns;
//node.Update(state.Winner == node.LastToMove ? 1.0 - fastWinReward : 0.0);
node.Update(state.Winner == node.LastToMove ? 1.0 : 0.0);
node = node.Parent;
visitedNodes++;
}
semaphore.Release();
});
// Cut Short
foreach (var child in rootNode.Children)
{
// If we have a potential move that has a 100% win rate and it's been visited a lot of times, stop searching
if (child.Visits > 500 && child.Wins == child.Visits)
{
forceEnd = true;
}
}
// Update Status
if (statusUpdate.ElapsedMilliseconds > millisecondsBetweenUpdates && OnStatus != null)
{
EngineStatus status = new EngineStatus();
foreach (var child in rootNode.Children)
{
double eval = Math.Round((child.Visits > 0 ? 200.0 * Math.Min(child.Wins, child.Visits) / child.Visits : 0) - 100.0, 2);
string pv = "";
var c = child;
while (c != null && c.Move >= 0 && c.Move <= 80)
{
pv += Constants.TileNames[c.Move] + " (" + Math.Round(c.Wins, 2) + "/" + c.Visits + ") ";
c = c.Children?.OrderBy(x => x.Visits)?.ThenBy(x => x.Wins)?.LastOrDefault();
}
status.Add(child?.Move ?? 80, eval, pv, child.Visits);
}
status.Sort();
OnStatus(this, status);
statusUpdate = Stopwatch.StartNew();
}
}
forceEnd = true;
return(rootNode.Children.OrderBy(x => x.Visits).LastOrDefault().Move);
}
private double UpperConfidenceBound(MonteCarloTreeSearchNode node)
{
return(node.Wins / node.Visits + Math.Sqrt(2.0 * Math.Log(Visits) / node.Visits));
}
private int MonteCarloTreeSearch(Board rootState)
{
var rootNode = new MonteCarloTreeSearchNode(rootState, GetMoves);
var forceWin = false;
while (!cancel.Cancelled && !forceWin)
{
var node = rootNode;
var state = new Board(rootState);
state.fastWinSearch = _allowFastWinSearch;
simulationCount++;
// Select
while (node.Untried.Count == 0 && node.Children.Count > 0)
{
node = node.SelectChild(_ucbFactor);
state.MakeMove(node.Move);
visitedNodes++;
}
// Expand
if (node.Untried.Count > 0)
{
var move = node.Untried[random.Next(node.Untried.Count)];
state.MakeMove(move);
node = node.AddChild(state, move);
visitedNodes++;
}
// Simulate
while (state.Winner == Player.Empty && state.Turn < VolcanoGame.Settings.TournamentAdjudicateMaxTurns)
{
var moves = state.GetMoves();
if (moves.Count == 0)
{
break;
}
state.MakeMove(moves[random.Next(moves.Count)]);
visitedNodes++;
}
// Backpropagate
while (node != null)
{
node.Update(state.Winner == node.LastToMove ? 1.0 : 0.0);
node = node.Parent;
visitedNodes++;
}
// Cut Short
if (_allowForcedWins)
{
foreach (var child in rootNode.Children)
{
// If we have a potential move that has a 100% win rate and it's been visited a lot of times, stop searching
if (child.Visits > 500 && child.Wins == child.Visits)
{
forceWin = true;
}
}
}
ReportStatus(rootNode);
}
ReportStatus(rootNode);
return(rootNode.Children.OrderBy(x => x.Visits).LastOrDefault().Move);
}
private int MonteCarloTreeSearch(Board rootState)
{
var rootNode = new MonteCarloTreeSearchNode(rootState, GetMoves);
var forceWin = false;
while (!cancel.Cancelled && !forceWin)
{
var node = rootNode;
var state = new Board(rootState);
state.allowHash = _allowHash;
state.fastWinSearch = _allowFastWinSearch;
if (_allowHash)
{
state.winHashes = winHashes;
}
simulationCount++;
// Select
while (node.Untried.Count == 0 && node.Children.Count > 0)
{
node = node.SelectChild(_ucbFactor);
state.MakeMove(node.Move);
visitedNodes++;
}
// Expand
if (node.Untried.Count > 0)
{
var move = node.Untried[random.Next(node.Untried.Count)];
state.MakeMove(move);
node = node.AddChild(state, move);
visitedNodes++;
}
// Simulate
while (state.Winner == Player.Empty && state.Turn < VolcanoGame.Settings.TournamentAdjudicateMaxTurns)
{
var moves = state.GetMoves();
if (moves.Count == 0)
{
break;
}
state.MakeMove(moves[random.Next(moves.Count)]);
visitedNodes++;
}
// Backpropagate
while (node != null)
{
node.Update(state.Winner == node.LastToMove ? 1.0 : 0.0);
node = node.Parent;
visitedNodes++;
}
// Cut Short
if (_allowForcedWins)
{
foreach (var child in rootNode.Children)
{
// If we have a potential move that has a 100% win rate and it's been visited a lot of times, stop searching
if (child.Visits > 500 && child.Wins == child.Visits)
{
forceWin = true;
}
}
}
// Update Status
if (statusUpdate.ElapsedMilliseconds > millisecondsBetweenUpdates && OnStatus != null)
{
EngineStatus status = new EngineStatus();
foreach (var child in rootNode.Children)
{
double eval = Math.Round((child.Visits > 0 ? 200.0 * child.Wins / child.Visits : 0) - 100.0, 2);
string pv = "";
var c = child;
while (c != null && c.Move >= 0 && c.Move <= 80)
{
pv += Constants.TileNames[c.Move] + " (" + c.Wins + "/" + c.Visits + ") ";
c = c.Children?.OrderBy(x => x.Visits)?.ThenBy(x => x.Wins)?.LastOrDefault();
}
status.Add(child?.Move ?? 80, eval, pv, child.Visits);
}
status.Sort();
OnStatus?.Invoke(this, status);
statusUpdate = Stopwatch.StartNew();
}
}
return(rootNode.Children.OrderBy(x => x.Visits).LastOrDefault().Move);
}
