/// <summary>
/// Simulates the playout till the end and calculates a reward.
/// </summary>
public static float DefaultPolicy(GameInstance game, TeamColor startingTeam)
{
if (game.IsFinished)
{
Debug.Assert(game.VictoryTeam.HasValue || game.AllDead, "game.VictoryTeam.HasValue");
return(CalculateDeltaReward(game, startingTeam, game.VictoryTeam));
}
Debug.Assert(game.CurrentTeam.HasValue, "game.CurrentTeam.HasValue");
var copy = game.CopyStateOnly();
const int maxDefaultPolicyIterations = 200;
int iterations = maxDefaultPolicyIterations;
ReplayRecorder.Instance.Clear();
bool wasMove = false;
while (!copy.IsFinished && iterations-- > 0)
{
var action = ActionGenerator.DefaultPolicyAction(copy);
if (action.Type == UctActionType.Move)
{
if (wasMove)
{
action = UctAction.EndTurnAction();
}
wasMove = true;
}
if (action.Type == UctActionType.EndTurn)
{
wasMove = false;
}
if (action.Type == UctActionType.Null)
{
throw new InvalidOperationException();
}
ActionEvaluator.FNoCopy(copy, action);
}
if (iterations <= 0)
{
ReplayRecorder.Instance.SaveAndClear(game, 0);
//throw new InvariantViolationException("MCTS playout timeout");
Utils.Log(LogSeverity.Error, nameof(UctAlgorithm),
$"DefaultPolicy ran out of time (over {maxDefaultPolicyIterations} iterations for playout), computed results are likely wrong.");
return(0);
}
TeamColor?victoryTeam = copy.VictoryTeam;
return(CalculateDeltaReward(game, startingTeam, victoryTeam));
}
/// <summary>
/// Returns a list of best possible actions until the end of a turn.
/// </summary>
private List <UctAction> SelectBestActions(UctNode root)
{
var result = new List <UctAction>();
UctNode current = root;
do
{
if (current.Children.Count == 0)
{
break;
}
UctNode max = current.Children.FastMax(c => c.Q / c.N);
if (max.Q / max.N < 0.2)
{
var state = current.State.CopyStateOnly();
do
{
var action = ActionGenerator.RuleBasedAction(state);
state = ActionEvaluator.F(state, action);
if (action.Type == UctActionType.EndTurn)
{
goto done;
}
else
{
result.Add(action);
if (action.Type == UctActionType.DefensiveMove)
{
goto done;
}
}
} while (true);
}
if (max.Action.Type != UctActionType.EndTurn)
{
if (max.IsTerminal)
{
//Console.WriteLine("Found terminal");
}
result.Add(max.Action);
}
current = max;
} while (current.Action.Type != UctActionType.EndTurn);
done:
return(result);
}
public async Task SlowPlayTurn(GameEventHub eventHub)
{
do
{
var possibleActions = ActionGenerator.PossibleActions(_gameInstance, null, true, true);
var chosenAction = possibleActions[Generator.Random.Next(possibleActions.Count)];
if (chosenAction.Type == UctActionType.EndTurn)
{
break;
}
await eventHub.SlowPlayAction(_gameInstance, chosenAction);
} while (!_gameInstance.IsFinished);
}
public void FastPlayTurn(GameEventHub eventHub)
{
do
{
var possibleActions = ActionGenerator.PossibleActions(_gameInstance, null, true, true);
var chosenAction = possibleActions[Generator.Random.Next(possibleActions.Count)];
if (chosenAction.Type == UctActionType.EndTurn)
{
break;
}
ActionEvaluator.FNoCopy(_gameInstance, chosenAction);
} while (!_gameInstance.IsFinished);
}
public void PrecomputePossibleActions(bool allowMove, bool allowEndTurn)
{
Debug.Assert(!State.IsFinished, "!State.IsFinished");
if (PossibleActions == null)
{
if (Action.Type == UctActionType.DefensiveMove)
{
PossibleActions = new List <UctAction> {
UctAction.EndTurnAction()
};
}
else
{
PossibleActions = ActionGenerator.PossibleActions(State, this, allowMove, allowEndTurn);
}
}
}
public async Task SlowPlayTurn(GameEventHub eventHub)
{
do
{
var action = ActionGenerator.RuleBasedAction(_gameInstance);
if (action.Type == UctActionType.EndTurn)
{
break;
}
await eventHub.SlowPlayAction(_gameInstance, action);
if (action.Type == UctActionType.DefensiveMove)
{
break;
}
} while (!_gameInstance.IsFinished);
}
public void FastPlayTurn(GameEventHub eventHub)
{
do
{
var action = ActionGenerator.RuleBasedAction(_gameInstance);
if (action.Type == UctActionType.EndTurn)
{
break;
}
ActionEvaluator.FNoCopy(_gameInstance, action);
if (action.Type == UctActionType.DefensiveMove)
{
break;
}
} while (!_gameInstance.IsFinished);
}
