public void Expand(IPatchworkServerClient client)
{
#if DEBUG
if (Children.Count != 0)
{
throw new Exception("Cannot expand an already expanded node");
}
if (IsGameEnd)
{
throw new Exception("Cannot expand a GameEnd node");
}
#endif
//Advance
{
var node = MonteCarloTreeSearchAlphaZero.NodePool.Value.Get();
State.CloneTo(node.State);
node.State.Fidelity = SimulationFidelity.NoPiecePlacing;
node.State.PerformAdvanceMove();
node.Parent = this;
node.PieceToPurchase = null;
node.NetworkChildIndex = 0;
Children.Add(node);
}
//Purchase
for (var i = 0; i < 3; i++)
{
//This cares if they can actually place the piece only when expanding the root node
if (Helpers.ActivePlayerCanPurchasePiece(State, Helpers.GetNextPiece(State, i)))
{
var node = MonteCarloTreeSearchAlphaZero.NodePool.Value.Get();
State.CloneTo(node.State);
node.State.Fidelity = SimulationFidelity.NoPiecePlacing;
var pieceIndex = node.State.NextPieceIndex + i;
node.State.PerformPurchasePiece(pieceIndex);
node.Parent = this;
node.PieceToPurchase = pieceIndex;
node.NetworkChildIndex = i + 1;
Children.Add(node);
}
}
//Evaluate the networks of our children
var req = new EvaluateRequest();
for (var i = 0; i < Children.Count; i++)
{
req.State.Add(GameStateFactory.CreateGameState(Children[i].State));
}
var res = client.Evaluate(req);
for (var i = 0; i < Children.Count; i++)
{
Children[i].NetworkResult = res.Evaluations[i];
}
}
/// <summary>
/// Performs a MCTS search starting at the given state.
/// Returns the root of the search tree, you must call NodePool.Value.ReturnAll() afterwards.
/// </summary>
internal MCTSAZNode PerformMCTS(SimulationState state)
{
var root = NodePool.Value.Get();
state.CloneTo(root.State);
var req = new EvaluateRequest();
req.State.Add(GameStateFactory.CreateGameState(root.State));
var res = _client.Evaluate(req);
root.NetworkResult = res.Evaluations[0];
for (var i = 0; i < Iterations; i++)
{
//Selection
var leaf = Select(root);
float leafValue;
if (leaf.IsGameEnd)
{
//Value at a leaf is from the perspective of the player who performed an action to move the simulation to that state
leafValue = (leaf.State.WinningPlayer == leaf.Parent.State.ActivePlayer) ? 1 : -1;
}
else
{
//Expansion
Expand(leaf);
//Randomly choose one of the newly expanded nodes
leaf = Select(leaf);
//Simulation
leafValue = leaf.NetworkResult.WinRate;
}
//Backpropagation
do
{
leaf.ReceiveBackpropagation(leafValue);
leaf = leaf.Parent;
//Value at a leaf is from the perspective of the player who performed an action to move the simulation to that state
//When the player changes we need to negative the value so it is from the other players perspective
if (leaf != null && leaf.Parent != null && leaf.Parent.State.ActivePlayer != leaf.State.ActivePlayer)
{
leafValue = -leafValue;
}
} while (leaf != null);
}
return(root);
}
internal TrainSample MakeMoveWithResult(SimulationState state)
{
var res = new TrainSample();
res.State = GameStateFactory.CreateGameState(state);
var root = _mcts.PerformMCTS(state);
var bestChild = _mcts.FindBestChild(root);
//DumpChildren(root);
if (bestChild.PieceToPurchase.HasValue)
{
state.PerformPurchasePiece(bestChild.PieceToPurchase.Value);
}
else
{
state.PerformAdvanceMove();
}
//Add all the moves, with 0 if they aren't possible
res.MoveRating.Add(root.Children[0].VisitCount / (float)root.VisitCount);
for (var i = 0; i < 3; i++)
{
bool foundIt = false;
for (var j = 0; j < root.Children.Count; j++)
{
if (root.Children[j].PieceToPurchase == i)
{
foundIt = true;
res.MoveRating.Add(root.Children[j].VisitCount / (float)root.VisitCount);
break;
}
}
if (!foundIt)
{
res.MoveRating.Add(0);
}
}
MonteCarloTreeSearchAlphaZero.NodePool.Value.ReturnAll();
return(res);
}