/// <summary> /// Saves the last state to a file. /// </summary> /// <param name="gameState">The GameState.</param> private void SaveLastState(IGameState gameState) { FileStream fs = new FileStream(this.lastStateFile, FileMode.Create); BinaryWriter bw = new BinaryWriter(fs); for (int row = 0; row < gameState.Height; ++row) { for (int col = 0; col < gameState.Width; ++col) { Location location = new Location(row, col); ushort position = location.ToUShort(gameState.Width); if (this.lastStates[position] != null) { State state = GetState(gameState, location); //file format: //ulong (old state) //byte (action) //ulong (new state) foreach (StateAction sa in this.lastStates[position]) { bw.Write(sa.State.Value); bw.Write((byte)sa.Action); bw.Write(state.Value); } } } } bw.Close(); fs.Close(); }
/// <summary> /// Gets the reward by going from <paramref name="oldLocation"/> to <paramref name="newLocation"/> /// by doing some action <paramref name="a"/> for a 15x15 or 20x20 map. /// </summary> /// <param name="gameState">The GameState.</param> /// <param name="oldLocation">The location before taking the action.</param> /// <param name="a">The action that was taken.</param> /// <param name="newLocation">The location after taking the action.</param> /// <returns>The reward by doing action <paramref name="a"/> from Location /// <paramref name="oldLocation"/> to get to Location <paramref name="newLocation"/> /// for a 15x15 and 20x20 map.</returns> private float GetReward15And20(IGameState gameState, Location oldLocation, Action a, Location newLocation) { float reward = 0; //give reward for getting food (more food => higher reward) reward += 0.5f * NumOfFoodNextTo(gameState, newLocation); //negative reward for having more than one ant walking to the same location, if (this.lastStates[newLocation.ToUShort(gameState.Width)].Count >= 2) { //but do not give that penalty when the ant did not move that turn //(because then that ant did nothing wrong). if (a != Action.None) { reward += -3.0f; } } return reward; }
/// <summary> /// Uses the lastStates to process the rewards from moves taken in the last turn. /// </summary> /// <param name="gameState">The GameState.</param> private void ProcessRewards(IGameState gameState) { //make sure the lastStates array is initialized, because it is not in the first turn. //(we cannot initialize it at the start of the turn because we need it then, and we cannot //initialize it before the first turn, because then we do not know the Width and Height of the map) if (this.lastStates != null) { for (int row = 0; row < gameState.Height; ++row) { for (int col = 0; col < gameState.Width; ++col) { Location newLocation = new Location(row, col); ushort newPosition = newLocation.ToUShort(gameState.Width); if (this.lastStates[newPosition] != null) { State state = GetState(gameState, newLocation); foreach (StateAction sa in this.lastStates[newPosition]) { int oldPosition = sa.State.GetPosition(); Location oldLocation = new Location(oldPosition / gameState.Width, oldPosition % gameState.Width); float reward = GetReward(gameState, oldLocation, sa.Action, newLocation); this.learn.ProcessReward(reward, sa.State, state, sa.Action, this.alpha, this.gamma); } } } } } }
/// <summary> /// Gets a State object representing the current state. /// </summary> /// <param name="state">The GameState.</param> /// <param name="location">The Location from which to generate a state.</param> /// <returns>A State object that repressents the current state given the location.</returns> public State GetState(IGameState gameState, Location location) { QTile[] tiles = new QTile[statePositions.Length]; for (int i = 0; i < statePositions.Length; ++i) { Location loc = gameState.GetDestination(location, statePositions[i]); tiles[i] = gameState[loc].ToQTile(); } ushort position = location.ToUShort(gameState.Width); return new State(tiles, position); }