public static GameState Tick(GameState gameState, PlayerInput playerInput)
{
if (playerInput.Player != gameState.NextPlayer)
{
throw new ArgumentException("Attempted to move out-of-turn");
}
BoardState newBoardState = BoardState.MakeMove(gameState.BoardState, playerInput.Player, playerInput.X, playerInput.Y);
BoardState.Player newNextPlayer = NextPlayer(gameState.NextPlayer);
BoardState.Winner newWinner = BoardState.CheckForWinner(newBoardState);
GameState newGameState = new GameState(newBoardState, newNextPlayer, newWinner);
return(newGameState);
}
public void UpdateQ(BoardState oldBoardState, PlayerInput oldAction, BoardState newBoardState)
{
BoardState.Winner winner = BoardState.CheckForWinner(newBoardState);
PlayerInput[] availableNewActions;
if (winner != BoardState.Winner.None)
{
availableNewActions = null;
}
else
{
availableNewActions = FindAvailableActions(newBoardState, oldAction.Player).ToArray();
}
UpdateQ(oldBoardState, oldAction, newBoardState, availableNewActions, winner);
}
private static float WinnerToReward(BoardState.Winner winner, BoardState.Player player)
{
switch (winner)
{
case BoardState.Winner.None:
return(0.0f);
case BoardState.Winner.Draw:
return(0.5f);
case BoardState.Winner.Player1:
return((player == BoardState.Player.Player1) ? 1.0f : -1.0f);
case BoardState.Winner.Player2:
return((player == BoardState.Player.Player2) ? 1.0f : -1.0f);
default:
throw new NotImplementedException();
}
}
public void UpdateQ(BoardState oldBoardState, PlayerInput oldAction, BoardState newBoardState, PlayerInput[] newActions, BoardState.Winner newBoardStateWinner)
{
float reward = WinnerToReward(newBoardStateWinner, oldAction.Player);
float oldQ = ReadQ(oldBoardState, oldAction);
float maxNewQ;
if (newBoardStateWinner != BoardState.Winner.None)
{
maxNewQ = WinnerToReward(newBoardStateWinner, oldAction.Player);
}
else
{
maxNewQ = FindMaxQ(newBoardState, newActions);
}
float updatedOldQ = oldQ + LearningRate * (reward + DiscountFactor * maxNewQ - oldQ);
WriteQ(oldBoardState, oldAction, updatedOldQ);
}
public GameState(BoardState boardState, BoardState.Player nextPlayer, BoardState.Winner winner)
{
BoardState = boardState;
NextPlayer = nextPlayer;
Winner = winner;
}
