/// <summary>
/// Generates possible direct ability use actions.
/// </summary>
public static bool GenerateDirectAbilityUse(GameInstance state,
CachedMob mob,
List <UctAction> result)
{
bool foundAbilityUse = false;
var mobInfo = mob.MobInfo;
var mobId = mob.MobId;
foreach (var abilityId in mobInfo.Abilities)
{
if (!GameInvariants.IsAbilityUsableNoTarget(state, mobId, abilityId))
{
continue;
}
foreach (var targetId in state.MobManager.Mobs)
{
if (GameInvariants.IsAbilityUsable(state, mob, state.CachedMob(targetId), abilityId))
{
foundAbilityUse = true;
var action = UctAction.AbilityUseAction(abilityId, mobId, targetId);
GameInvariants.AssertValidAction(state, action);
result.Add(action);
}
}
}
return(foundAbilityUse);
}
public UctNode(float q, int n, UctAction action, GameInstance state)
{
Id = _id++;
Q = q;
N = n;
Action = action;
State = state;
}
/// <summary>
/// Runs a playout with two given controllers and reports the result.
/// </summary>
public static PlayoutResult Playout(GameInstance game, IMobController ai1, IMobController ai2)
{
var hub = new GameEventHub(game);
game.MobManager.Teams[TeamColor.Red] = ai1;
game.MobManager.Teams[TeamColor.Blue] = ai2;
const int maxIterations = 100;
int i = 0;
for (; i < maxIterations && !game.IsFinished; i++)
{
game.CurrentController.FastPlayTurn(hub);
ActionEvaluator.FNoCopy(game, UctAction.EndTurnAction());
}
float totalMaxHp = 0;
float totalCurrentHp = 0;
foreach (var mobId in game.MobManager.Mobs)
{
totalMaxHp += game.MobManager.MobInfos[mobId].MaxHp;
totalCurrentHp += Math.Max(0, game.State.MobInstances[mobId].Hp);
}
int red = 0;
int blue = 0;
Utils.Log(LogSeverity.Error, nameof(GameEvaluator),
$"Playout time limit reached at {maxIterations} rounds");
if (i < maxIterations && game.VictoryTeam.HasValue)
{
if (game.VictoryTeam.Value == TeamColor.Red)
{
red++;
}
else
{
blue++;
}
Accounting.IncrementWinner(game.VictoryController);
}
var gamePercentage = totalCurrentHp / totalMaxHp;
Debug.Assert(gamePercentage >= 0);
var mobsCount = game.MobManager.Mobs.Count;
var dis = new Normal(mobsCount * 2, mobsCount);
dis.Density(mobsCount * 2);
return(new PlayoutResult(i, gamePercentage, game.State.AllPlayed, i == maxIterations, red, blue));
}
/// <summary>
/// Runs the actual MCTS search on a given initial state.
/// </summary>
public UctSearchResult UctSearch(GameInstance initialState)
{
var root = new UctNode(0, 0, UctAction.NullAction(), initialState.CopyStateOnly());
if (!initialState.CurrentTeam.HasValue)
{
throw new ArgumentException("Trying to do UCT search on a finished game.");
}
var stopwatch = new Stopwatch();
stopwatch.Start();
var iterationStopwatch = new Stopwatch();
var startingTeam = initialState.CurrentTeam.Value;
int iterations = 0;
do
{
iterations++;
TotalIterationCount++;
iterationStopwatch.Restart();
OneIteration(root, startingTeam);
iterationStopwatch.Stop();
MillisecondsPerIterationAverage.Add(iterationStopwatch.Elapsed.TotalMilliseconds);
if (_iterationsOverTime)
{
if (iterations >= _thinkTime)
{
break;
}
}
else
{
if (stopwatch.ElapsedMilliseconds >= _thinkTime)
{
break;
}
}
} while (true);
stopwatch.Stop();
#if DOTGRAPH
UctDebug.PrintTreeRepresentation(root);
#endif
Interlocked.Increment(ref SearchCount);
var actions = SelectBestActions(root);
var millisecondsPerIteration = (double)stopwatch.ElapsedMilliseconds / (double)iterations;
return(new UctSearchResult(actions, millisecondsPerIteration));
}
/// <summary>
/// Simulates the playout till the end and calculates a reward.
/// </summary>
public static float DefaultPolicy(GameInstance game, TeamColor startingTeam)
{
if (game.IsFinished)
{
Debug.Assert(game.VictoryTeam.HasValue || game.AllDead, "game.VictoryTeam.HasValue");
return(CalculateDeltaReward(game, startingTeam, game.VictoryTeam));
}
Debug.Assert(game.CurrentTeam.HasValue, "game.CurrentTeam.HasValue");
var copy = game.CopyStateOnly();
const int maxDefaultPolicyIterations = 200;
int iterations = maxDefaultPolicyIterations;
ReplayRecorder.Instance.Clear();
bool wasMove = false;
while (!copy.IsFinished && iterations-- > 0)
{
var action = ActionGenerator.DefaultPolicyAction(copy);
if (action.Type == UctActionType.Move)
{
if (wasMove)
{
action = UctAction.EndTurnAction();
}
wasMove = true;
}
if (action.Type == UctActionType.EndTurn)
{
wasMove = false;
}
if (action.Type == UctActionType.Null)
{
throw new InvalidOperationException();
}
ActionEvaluator.FNoCopy(copy, action);
}
if (iterations <= 0)
{
ReplayRecorder.Instance.SaveAndClear(game, 0);
//throw new InvariantViolationException("MCTS playout timeout");
Utils.Log(LogSeverity.Error, nameof(UctAlgorithm),
$"DefaultPolicy ran out of time (over {maxDefaultPolicyIterations} iterations for playout), computed results are likely wrong.");
return(0);
}
TeamColor?victoryTeam = copy.VictoryTeam;
return(CalculateDeltaReward(game, startingTeam, victoryTeam));
}
/// <summary>
/// Generates a list of possible actions, truncated for the purposes of MCTS.
/// </summary>
public static List <UctAction> PossibleActions(GameInstance game, UctNode parent, bool allowMove,
bool allowEndTurn)
{
var result = new List <UctAction>(10);
var currentMob = game.CurrentMob;
if (currentMob.HasValue)
{
var mob = game.CachedMob(currentMob.Value);
GameInvariants.AssertMobPlayable(game, mob);
bool foundAbilityUse = GenerateDirectAbilityUse(game, mob,
result);
// We disable movement if there is a possibility to cast abilities.
if (allowMove && (Constants.AlwaysAttackMove || !foundAbilityUse))
{
GenerateAttackMoveActions(game, game.CachedMob(mob.MobId), result);
}
if (allowMove)
{
if (parent == null || parent.Action.Type != UctActionType.DefensiveMove)
{
GenerateDefensiveMoveActions(game, mob, result);
}
}
}
else
{
Utils.Log(LogSeverity.Warning, nameof(UctNode),
"Final state reached while trying to compute possible actions.");
throw new InvalidOperationException();
}
if (allowEndTurn)
{
// We would skip end turn if there are not enough actions.
if (!Constants.EndTurnAsLastResort || result.Count <= 1)
{
result.Add(UctAction.EndTurnAction());
}
}
GameInvariants.AssertValidActions(game, result);
return(result);
}
/// <summary>
/// Generates a number of defensive move actions based on a heatmap.
/// </summary>
public static void GenerateDefensiveMoveActions(GameInstance state, CachedMob mob, List <UctAction> result)
{
var heatmap = Heatmap.BuildHeatmap(state, null, false);
var coords = new List <AxialCoord>();
var mobInstance = mob.MobInstance;
var mobId = mob.MobId;
foreach (var coord in heatmap.Map.AllCoords)
{
if (heatmap.Map[coord] != heatmap.MinValue)
{
continue;
}
if (state.Map[coord] == HexType.Wall)
{
continue;
}
if (state.State.AtCoord(coord, true).HasValue)
{
continue;
}
bool canMoveTo = state.Pathfinder.Distance(mobInstance.Coord, coord) <= mobInstance.Ap;
if (!canMoveTo)
{
continue;
}
coords.Add(coord);
}
coords.Shuffle();
int maximumMoveActions = Math.Max(0, 3 - result.Count);
for (int i = 0; i < Math.Min(coords.Count, maximumMoveActions); i++)
{
var action = UctAction.DefensiveMoveAction(mobId, coords[i]);
GameInvariants.AssertValidAction(state, action);
result.Add(action);
}
}
private static UctAction DeterministicMaxAbilityRatio(GameInstance game, List <UctAction> actions)
{
UctAction max = actions[0];
var maxAbilityInfo = game.MobManager.Abilities[max.AbilityId];
for (int i = 1; i < actions.Count; i++)
{
var abilityInfo = game.MobManager.Abilities[actions[i].AbilityId];
if (abilityInfo.DmgCostRatio > maxAbilityInfo.DmgCostRatio)
{
max = actions[i];
maxAbilityInfo = abilityInfo;
}
}
return(max);
}
public void PrecomputePossibleActions(bool allowMove, bool allowEndTurn)
{
Debug.Assert(!State.IsFinished, "!State.IsFinished");
if (PossibleActions == null)
{
if (Action.Type == UctActionType.DefensiveMove)
{
PossibleActions = new List <UctAction> {
UctAction.EndTurnAction()
};
}
else
{
PossibleActions = ActionGenerator.PossibleActions(State, this, allowMove, allowEndTurn);
}
}
}
/// <summary>
/// Calculates a move towards an enemy, or ends a turn if no such move is possible.
/// </summary>
private static UctAction PickMoveTowardsEnemyAction(GameInstance game, CachedMob mob,
CachedMob possibleTarget)
{
foreach (var targetId in game.MobManager.Mobs)
{
var target = game.CachedMob(targetId);
if (!GameInvariants.IsTargetableNoSource(game, mob, target))
{
continue;
}
var action = FastMoveTowardsEnemy(game, mob, target);
if (action.Type == UctActionType.Move)
{
return(action);
}
}
return(UctAction.EndTurnAction());
}
/// <summary>
/// Calculates an action based on a simple set of rules.
/// </summary>
public static UctAction RuleBasedAction(GameInstance game)
{
if (Constants.FastActionGeneration)
{
return(DefaultPolicyAction(game));
}
var result = new List <UctAction>();
var currentMob = game.CurrentMob;
if (!currentMob.HasValue)
{
return(UctAction.EndTurnAction());
}
var mob = game.CachedMob(currentMob.Value);
GenerateDirectAbilityUse(game, mob, result);
if (result.Count > 0)
{
return(MaxAbilityRatio(game, result));
}
GenerateAttackMoveActions(game, mob, result);
if (result.Count > 0)
{
return(MaxAbilityRatio(game, result));
}
GenerateDefensiveMoveActions(game, mob, result);
if (result.Count > 0)
{
return(result[0]);
}
return(UctAction.EndTurnAction());
}
/// <summary>
/// Returns an action that moves towards an enemy as fast as possible.
/// </summary>
public static UctAction FastMoveTowardsEnemy(GameInstance state, CachedMob mob, CachedMob target)
{
var pathfinder = state.Pathfinder;
var moveTarget = pathfinder.FurthestPointToTarget(mob, target);
if (moveTarget != null && pathfinder.Distance(mob.MobInstance.Coord, moveTarget.Value) <=
mob.MobInstance.Ap)
{
return(UctAction.MoveAction(mob.MobId, moveTarget.Value));
}
else if (moveTarget == null)
{
// Intentionally doing nothing
return(UctAction.EndTurnAction());
}
else
{
Utils.Log(LogSeverity.Debug, nameof(AiRuleBasedController),
$"Move failed since target is too close, source {mob.MobInstance.Coord}, target {target.MobInstance.Coord}");
return(UctAction.EndTurnAction());
}
}
/// <summary>
/// Runs a playout with the given encounter defined by a DNA pair and both controllers.
/// </summary>
public static int Playout(GameInstance game, DNA d1, DNA d2, IMobController c1, IMobController c2)
{
GameSetup.OverrideGameDna(game, d1, d2);
game.AssignAiControllers(c1, c2);
int iterations = Constants.MaxPlayoutEvaluationIterations;
var hub = new GameEventHub(game);
while (!game.IsFinished && iterations-- > 0)
{
game.CurrentController.FastPlayTurn(hub);
ActionEvaluator.FNoCopy(game, UctAction.EndTurnAction());
}
if (Constants.GetLogBuffer().ToString().Length != 0)
{
Console.WriteLine(Constants.GetLogBuffer());
}
Constants.ResetLogBuffer();
return(Constants.MaxPlayoutEvaluationIterations - iterations);
}
/// <summary>
/// Generates possible attack move actions.
/// </summary>
public static void GenerateAttackMoveActions(GameInstance state, CachedMob mob, List <UctAction> result)
{
var mobInfo = mob.MobInfo;
var mobInstance = mob.MobInstance;
foreach (var enemyId in state.MobManager.Mobs)
{
var target = state.CachedMob(enemyId);
if (!GameInvariants.IsTargetableNoSource(state, mob, target))
{
continue;
}
AxialCoord myCoord = mobInstance.Coord;
AxialCoord?closestCoord = null;
int? distance = null;
int? chosenAbilityId = null;
foreach (var coord in state.Map.EmptyCoords)
{
if (!state.Map.IsVisible(coord, target.MobInstance.Coord))
{
continue;
}
var possibleMoveAction = GameInvariants.CanMoveTo(state, mob, coord);
if (possibleMoveAction.Type == UctActionType.Null)
{
continue;
}
Debug.Assert(possibleMoveAction.Type == UctActionType.Move);
foreach (var abilityId in mobInfo.Abilities)
{
if (!GameInvariants.IsAbilityUsableFrom(state, mob, coord, target, abilityId))
{
continue;
}
int myDistance = state.Pathfinder.Distance(myCoord, coord);
if (!closestCoord.HasValue)
{
chosenAbilityId = abilityId;
closestCoord = coord;
distance = myDistance;
}
else if (distance.Value > myDistance)
{
chosenAbilityId = abilityId;
closestCoord = coord;
distance = myDistance;
}
}
}
if (closestCoord.HasValue)
{
if (Constants.AttackMoveEnabled)
{
var action = UctAction.AttackMoveAction(mob.MobId,
closestCoord.Value,
chosenAbilityId.Value,
target.MobId);
var after = ActionEvaluator.F(state, action.ToPureMove());
GameInvariants.AssertValidAbilityUseAction(after, action.ToPureAbilityUse());
GameInvariants.AssertValidAction(state, action);
result.Add(action);
}
else
{
var action = UctAction.MoveAction(mob.MobId, closestCoord.Value);
GameInvariants.AssertValidAction(state, action);
result.Add(action);
}
}
}
}
public static UctNode FromAction(GameInstance game, UctAction action)
{
return(new UctNode(action, ActionEvaluator.F(game, action)));
}
public UctNode(UctAction action, GameInstance state) : this(0, 0, action, state)
{
}
public static float PlayoutAction(GameInstance game, UctAction action, TeamColor startingTeam)
{
return(DefaultPolicy(ActionEvaluator.F(game, action), startingTeam));
}
/// <summary>
/// Calculates an action according to a simple default policy. Used mainly
/// in MCTS playouts.
/// </summary>
public static UctAction DefaultPolicyAction(GameInstance state)
{
var mobId = state.CurrentMob;
if (mobId == null)
{
throw new InvalidOperationException("Requesting mob action when there is no current mob.");
}
Debug.Assert(state.State.MobInstances[mobId.Value].Hp > 0, "Current mob is dead");
var mob = state.CachedMob(mobId.Value);
if (mob.MobInstance.Ap == 0)
{
return(UctAction.EndTurnAction());
}
var abilityIds = new List <int>();
foreach (var possibleAbilityId in mob.MobInfo.Abilities)
{
if (GameInvariants.IsAbilityUsableNoTarget(state, mobId.Value, possibleAbilityId))
{
abilityIds.Add(possibleAbilityId);
}
}
int moveTargetId = MobInstance.InvalidId;
var actions = new List <UctAction>();
foreach (var possibleTargetId in state.MobManager.Mobs)
{
var possibleTarget = state.CachedMob(possibleTargetId);
moveTargetId = possibleTargetId;
if (!GameInvariants.IsTargetable(state, mob, possibleTarget))
{
continue;
}
if (abilityIds.Count == 0)
{
continue;
}
foreach (var abilityId in abilityIds)
{
if (GameInvariants.IsAbilityUsableApRangeCheck(state, mob, possibleTarget, abilityId))
{
actions.Add(UctAction.AbilityUseAction(abilityId, mob.MobId, possibleTargetId));
}
}
}
if (actions.Count > 0)
{
return(MaxAbilityRatio(state, actions));
}
if (moveTargetId != MobInstance.InvalidId)
{
return(PickMoveTowardsEnemyAction(state, state.CachedMob(mobId.Value),
state.CachedMob(moveTargetId)));
}
else
{
Utils.Log(LogSeverity.Error, nameof(ActionGenerator), "No targets, game should be over");
throw new InvalidOperationException("No targets, game should be over.");
}
}
