/// <summary>
/// Saves the last state to a file.
/// </summary>
/// <param name="gameState">The GameState.</param>
private void SaveLastState(IGameState gameState)
{
FileStream fs = new FileStream(this.lastStateFile, FileMode.Create);
BinaryWriter bw = new BinaryWriter(fs);
for (int row = 0; row < gameState.Height; ++row) {
for (int col = 0; col < gameState.Width; ++col) {
Location location = new Location(row, col);
ushort position = location.ToUShort(gameState.Width);
if (this.lastStates[position] != null) {
State state = GetState(gameState, location);
//file format:
//ulong (old state)
//byte (action)
//ulong (new state)
foreach (StateAction sa in this.lastStates[position]) {
bw.Write(sa.State.Value);
bw.Write((byte)sa.Action);
bw.Write(state.Value);
}
}
}
}
bw.Close();
fs.Close();
}
/// <summary>
/// Gets the reward by going from <paramref name="oldLocation"/> to <paramref name="newLocation"/>
/// by doing some action <paramref name="a"/> for a 15x15 or 20x20 map.
/// </summary>
/// <param name="gameState">The GameState.</param>
/// <param name="oldLocation">The location before taking the action.</param>
/// <param name="a">The action that was taken.</param>
/// <param name="newLocation">The location after taking the action.</param>
/// <returns>The reward by doing action <paramref name="a"/> from Location
/// <paramref name="oldLocation"/> to get to Location <paramref name="newLocation"/>
/// for a 15x15 and 20x20 map.</returns>
private float GetReward15And20(IGameState gameState, Location oldLocation, Action a, Location newLocation)
{
float reward = 0;
//give reward for getting food (more food => higher reward)
reward += 0.5f * NumOfFoodNextTo(gameState, newLocation);
//negative reward for having more than one ant walking to the same location,
if (this.lastStates[newLocation.ToUShort(gameState.Width)].Count >= 2) {
//but do not give that penalty when the ant did not move that turn
//(because then that ant did nothing wrong).
if (a != Action.None) {
reward += -3.0f;
}
}
return reward;
}
/// <summary>
/// Uses the lastStates to process the rewards from moves taken in the last turn.
/// </summary>
/// <param name="gameState">The GameState.</param>
private void ProcessRewards(IGameState gameState)
{
//make sure the lastStates array is initialized, because it is not in the first turn.
//(we cannot initialize it at the start of the turn because we need it then, and we cannot
//initialize it before the first turn, because then we do not know the Width and Height of the map)
if (this.lastStates != null) {
for (int row = 0; row < gameState.Height; ++row) {
for (int col = 0; col < gameState.Width; ++col) {
Location newLocation = new Location(row, col);
ushort newPosition = newLocation.ToUShort(gameState.Width);
if (this.lastStates[newPosition] != null) {
State state = GetState(gameState, newLocation);
foreach (StateAction sa in this.lastStates[newPosition]) {
int oldPosition = sa.State.GetPosition();
Location oldLocation = new Location(oldPosition / gameState.Width, oldPosition % gameState.Width);
float reward = GetReward(gameState, oldLocation, sa.Action, newLocation);
this.learn.ProcessReward(reward, sa.State, state, sa.Action, this.alpha, this.gamma);
}
}
}
}
}
}
/// <summary>
/// Gets a State object representing the current state.
/// </summary>
/// <param name="state">The GameState.</param>
/// <param name="location">The Location from which to generate a state.</param>
/// <returns>A State object that repressents the current state given the location.</returns>
public State GetState(IGameState gameState, Location location)
{
QTile[] tiles = new QTile[statePositions.Length];
for (int i = 0; i < statePositions.Length; ++i) {
Location loc = gameState.GetDestination(location, statePositions[i]);
tiles[i] = gameState[loc].ToQTile();
}
ushort position = location.ToUShort(gameState.Width);
return new State(tiles, position);
}
