Beispiel #1
0
 /// <summary>
 /// Construct a State-Action pair instance
 /// </summary>
 /// <param name="state">The state</param>
 /// <param name="action">The action</param>
 public SAPair(GameState state, Direction action) { this.State = state; this.Action = action; }
Beispiel #2
0
 /// <summary>
 /// Initiates the game-play ops
 /// </summary>
 private void Gameplay()
 {
     GameState gs = null;
     do
     {
         try
         {
             byte[] status = new byte[26];
             int rcv = this.Channel.Receive(status);
             if (rcv == 0) { o("!!Connection Dropped!!"); return; }
             gs = new GameState(status);
             Direction direction = this.__Leaner.ChooseAction(gs);
             this.Channel.Send(this.s2b(((int)direction).ToString()));
             o("Moved: {0}", direction.ToString());
         }
         catch (Exception e) { System.IO.File.AppendAllText("error.log", e.ToString()); o("Exception\n[ ABORT ]"); return; }
     } while (gs != null && gs.Game_State != GameState.State.GAME_FINISHED);
     o("==== Game Ended ====");
 }
Beispiel #3
0
 /// <summary>
 /// Check if the state is in front of a gate
 /// </summary>
 /// <param name="s">The game's state</param>
 /// <param name="opGate">`true` to check if the state is in front of opponent's gate; otherwise `false` to check if the state is in front of mine gate</param>
 /// <returns>True if state is in front of a gate</returns>
 public bool IsInFrontOfGate(GameState s, bool opGate = true)
 {
     var gColumn = 9;
     var g1 = new System.Drawing.Point(3, gColumn);
     var g2 = new System.Drawing.Point(4, gColumn);
     if (!opGate)
     {
         g1 = new System.Drawing.Point(3, 1);
         g2 = new System.Drawing.Point(4, 1);
     }
     return s.MyLocation == g1 || s.MyLocation == g2;
 }
Beispiel #4
0
 /// <summary>
 /// Executes ops and returns a movement direction based on game's status
 /// </summary>
 /// <param name="gameState">The game status</param>
 /// <returns>The direction</returns>
 public Direction ChooseAction(GameState gameState)
 {
     try
     {
         // Next-State Max Q Pair List
         List<KeyValuePair<KeyValuePair<float, float>, Direction>> nsmqpl = new List<KeyValuePair<KeyValuePair<float, float>, Direction>>();
         // Next-State All Possible Q Pair List
         List<KeyValuePair<KeyValuePair<float, float>, Direction>> nsAPqpl = new List<KeyValuePair<KeyValuePair<float, float>, Direction>>();
         // Next-State Max Q Pair
         KeyValuePair<KeyValuePair<float, float>, Direction> nsmqp = new KeyValuePair<KeyValuePair<float, float>, Direction>(new KeyValuePair<float, float>(float.NegativeInfinity, float.NegativeInfinity), Direction.HOLD);
         // foreach valid directions
         foreach (Direction direction in this.getValidDirections(gameState))
         {
             // fetch the Next-State Q Value.
             var nsqv = new KeyValuePair<float, float>(this.getQVal(gameState, direction), this.getReward(this.getNextState(gameState, direction), gameState));
             // current next-state pair
             var cnsp = new KeyValuePair<KeyValuePair<float, float>, Direction>(nsqv, direction);
             // add it to all possible next-state list
             nsAPqpl.Add(cnsp);
             // fetch Next-State Q Value's Value
             var nsqvV = nsqv.Key + nsqv.Value;
             // fetch Next-State Max Q Pair's Value
             var nsmqpV = nsmqp.Key.Key + nsmqp.Key.Value;
             // check for MAX value
             if ( nsqvV > nsmqpV)
             {
                 nsmqp = cnsp;
                 // in next `if` statement we will fall-back to
                 // adding currently updated `nsmqp` into `nsmqpl`.
                 // but for now we need to clear the list
                 nsmqpl.Clear();
             }
             // if current next-state is equal to MAX
             if (nsqvV == nsmqp.Key.Key + nsmqp.Key.Value)
                 // add it to MAX list too
                 nsmqpl.Add(cnsp);
         }
         // init the candidate index
         int candIndex = -1;
         // candidate container
         KeyValuePair<KeyValuePair<float, float>, Direction> candidate = new KeyValuePair<KeyValuePair<float, float>, Direction>();
         // while there is a item in list
         while (nsmqpl.Count != 0)
         {
             // pick a randon index
             lock (this.RandGen) candIndex = this.RandGen.Next(0, nsmqpl.Count);
             // pick a random candidate
             candidate = nsmqpl[candIndex];
             // validate the mutation factor
             // why the THRESHOLD is {10}? because we should give the agent the chance
             // if he wants to cross-pass the whole horizon with one direction without 
             // getting intruptted by mutation factor.
             if (this.getMutationVal(gameState, candidate.Value) < 10) goto __PROCEED;
             // remove the current
             nsmqpl.RemoveAt(candIndex);
         }
         /**
          * If we reach here it means me have failed to pick a direction
          * and we have fell into a action-loop and we need to pick a new
          * random action to break the action-loop.
          */
         // get an index from all possible state-action list
         lock (this.RandGen) candIndex = this.RandGen.Next(0, nsAPqpl.Count);
         // make the random choosen as candidate
         candidate = nsAPqpl[candIndex];
     __PROCEED:
         // update the mution factor for current state
         this.updateMutaionFactor(gameState, candidate.Value);
         // the New Q Value
         var nQv = this.getReward(gameState) + Learner.GAMMA * (candidate.Key.Key + candidate.Key.Value);
         // update the Q value
         this.updateQ(
             gameState,          // The state
             candidate.Value,    // The direction
             nQv                 // The update value
         );
         // any of players scored?
         if (gameState.Game_State == GameState.State.OPPONENT_SCORED ||
             gameState.Game_State == GameState.State.PLAYER_SCORED)
             // re-init previous mutation factor container
             this.PrevMutationfactor = new KeyValuePair<SAPair, uint>();
         // we don't need to create a full-stack property
         if (this.GameStates.Count > 0)
             // pop any previously pushed state
             this.GameStates.Pop();
         // push current game status
         this.GameStates.Push(gameState);
         //return the candidate's direction;
         return candidate.Value;
     }
     catch (Exception e) { Console.WriteLine(); Console.WriteLine(e.ToString()); Debug.WriteLine(e.ToString()); throw e; }
 }
Beispiel #5
0
 /// <summary>
 /// Calculates the distance to goal
 /// </summary>
 /// <param name="s">Game's status</param>
 /// <param name="opGate">Check distance to opponent's gate or mine?</param>
 /// <param name="myDist">If `true` calculates my distance to a gate; otherwise calculates the opponent's to a gate</param>
 /// <returns>The distance to gate</returns>
 protected float CalcDistanceToGate(GameState s, bool opGate = true, bool myDist = true)
 {
     var gColumn = 10;
     var g1 = new System.Drawing.Point(3, gColumn);
     var g2 = new System.Drawing.Point(4, gColumn);
     if (!opGate)
     {
         g1 = new System.Drawing.Point(3, 0);
         g2 = new System.Drawing.Point(4, 0);
     }
     var x = s.MyLocation.X;
     var y = s.MyLocation.Y;
     if (!myDist)
     {
         x = s.OpponentLocation.X;
         y = s.OpponentLocation.Y;
     }
     var g39 = (float)(Math.Abs(g1.X - x) + Math.Abs(g1.Y - y));
     var g49 = (float)(Math.Abs(g2.X - x) + Math.Abs(g2.Y - y));
     return Math.Min(g39, g49);
 }
Beispiel #6
0
 /// <summary>
 /// Calculates the distance to the ball
 /// </summary>
 /// <param name="s">Game's status</param>
 /// <returns>The distance to ball</returns>
 protected float CalcDistanceToBall(GameState s)
 {
     if (s.IsBallMine) return 0;
     var b = s.OpponentLocation;
     var x = s.MyLocation.X;
     var y = s.MyLocation.Y;
     var bd = (float)(Math.Abs(b.X - x) + Math.Abs(b.Y - y));
     return bd;
 }
Beispiel #7
0
 /// <summary>
 /// Updates the `Mutationfactor` table
 /// </summary>
 /// <param name="s">The game state</param>
 /// <param name="a">The action</param>
 /// <param name="val">The mutation factor's value</param>
 protected void updateMutaionFactor(GameState s, Direction a)
 {
     var key = new SAPair(s, a);
     uint val = 0;
     if (this.PrevMutationfactor.Key != null && this.PrevMutationfactor.Key.GetHashCode() == key.GetHashCode())
         val = this.PrevMutationfactor.Value + 1;
     this.PrevMutationfactor = new KeyValuePair<SAPair, uint>(key, val);
 }
Beispiel #8
0
 /// <summary>
 /// Get `Mutationfactor` for the state-action
 /// </summary>
 /// <param name="s">For the game state</param>
 /// <param name="a">For the action</param>
 /// <returns>The mutation factor's value</returns>
 protected uint getMutationVal(GameState s, Direction a)
 {
     if (this.PrevMutationfactor.Key == null) return 0;
     var key = new SAPair(s, a);
     if (this.PrevMutationfactor.Key.GetHashCode() == key.GetHashCode())
         return this.PrevMutationfactor.Value;
     return 0;
 }
Beispiel #9
0
        /// <summary>
        /// Calculates the `Q` value
        /// </summary>
        /// <param name="s">For the game state</param>
        /// <param name="a">For the action</param>
        /// <returns>The `Q` value</returns>
        protected float getQVal(GameState s, Direction a)
        {
            var key = new SAPair(s, a).GetHashCode();
#if __USE_PREPROCESSED_Q__
            if (!this._Q.ContainsKey(key))
            {
                float q = 0;
                var ns = this.getNextState(s, a);
                if (a == Direction.EAST)
                {
                    if (ns.IsBallMine) q += 5.0F;
                    else q -= 5.0F;
                }
                if (ns.Game_State == GameState.State.OPPONENT_SCORED) q = float.NegativeInfinity;
                if (ns.Game_State == GameState.State.PLAYER_SCORED) q = float.PositiveInfinity;
                this.updateQ(s, a, q);
            }
            // if making an own-goal
            if (this.IsInFrontOfGate(s, false) && s.IsBallMine)
            {
                if(a == Direction.WEST)
                    this.updateQ(s, a, float.NegativeInfinity);
            }
            if (this.IsInFrontOfGate(s, true) && s.IsBallMine)
            {
                switch (a)
                {
                    case Direction.EAST: this.updateQ(s, a, float.PositiveInfinity); break;
                    case Direction.HOLD: this.updateQ(s, a, +100); break;
                    default: this.updateQ(s, a, float.NegativeInfinity); break;
                }
            }
#endif
            if (this._Q.ContainsKey(key))
                return _Q[key];
            return 0;
        }
Beispiel #10
0
 /// <summary>
 /// Updates the `Q` table
 /// </summary>
 /// <param name="s">The game state</param>
 /// <param name="a">The action</param>
 /// <param name="val">The value of state-action</param>
 protected void updateQ(GameState s, Direction a, float val)
 {
     var key = new SAPair(s, a).GetHashCode();
     if (this._Q.ContainsKey(key))
         this._Q[key] = val;
     else
         this._Q.Add(key, val);
 }
Beispiel #11
0
 /// <summary>
 /// Get next state of the game based on an action
 /// </summary>
 /// <param name="s">Current state of the game</param>
 /// <param name="a">The action to apply on current state</param>
 /// <returns>The next state</returns>
 protected GameState getNextState(GameState s, Direction a)
 {
     // make a clone of current status
     GameState _s = s.Clone() as GameState;
     /**
      * Update the location of mine by choosing the action
      */
     switch (a)
     {
         case Direction.HOLD: /* return currently cloned state */ return _s;
         case Direction.NORTH:
             _s.MyLocation = new System.Drawing.Point(_s.MyLocation.X - 1, s.MyLocation.Y);
             break;
         case Direction.SOUTH:
             _s.MyLocation = new System.Drawing.Point(_s.MyLocation.X + 1, s.MyLocation.Y);
             break;
         case Direction.WEST:
             _s.MyLocation = new System.Drawing.Point(_s.MyLocation.X, s.MyLocation.Y - 1);
             break;
         case Direction.EAST:
             _s.MyLocation = new System.Drawing.Point(_s.MyLocation.X, s.MyLocation.Y + 1);
             break;
     }
     // update the status of ball
     if (_s.MyLocation == _s.OpponentLocation) _s.IsBallMine = !_s.IsBallMine;
     /**
      * Update the scores status
      */
     if (_s.IsBallMine)
     {
         if (this.CalcDistanceToGate(s, true, true) == 0) { _s.MyScore++; _s.Game_State = GameState.State.PLAYER_SCORED; }
         else if (this.CalcDistanceToGate(s, false, true) == 0) { _s.OpponentScore++; _s.Game_State = GameState.State.OPPONENT_SCORED; }
     }
     else
     {
         if (this.CalcDistanceToGate(s, false, false) == 0) { _s.OpponentScore++; _s.Game_State = GameState.State.OPPONENT_SCORED; }
         else if (this.CalcDistanceToGate(s, true, false) == 0) { _s.MyScore++; _s.Game_State = GameState.State.PLAYER_SCORED; }
     }
     return _s;
 }
Beispiel #12
0
 /// <summary>
 /// Get reward of passed game status
 /// </summary>
 /// <param name="gs">The game state</param>
 /// <param name="prevgs">[OPTIONAL] The previous game state; If not provided the top of stack will be used!</param>
 /// <returns>The related reward for passed game state</returns>
 protected float getReward(GameState gs, GameState prevgs = null)
 {
     // if no previously game state has been recorded
     if (this.GameStates.Count == 0) /* NO REWARD AT INIT STATE */ return 0;
     // flag that we have peeked prev-gs from stack
     bool STACK_PEEKED = (prevgs == null);
     // if no previous game state has been provided? select the top of the stack
     if (prevgs == null) prevgs = this.GameStates.Peek();
     // get reward of current state based on combination of current and previous state's status
     float r = 0;
     // +100 if i score-up
     if (gs.Game_State == GameState.State.PLAYER_SCORED) r += 100.0F;
     // -100 if i score-down
     if (gs.Game_State == GameState.State.OPPONENT_SCORED) r -= 100.0F;
     // dynamic reward if i acquired the bull
     if (!prevgs.IsBallMine && gs.IsBallMine)
         // it is good to catch the ball nearby the friendly gate
         // to prevent the opponent to make a goal
         r += 50.0F / this.CalcDistanceToGate(gs, false);
     // dynamic reward if i lost the ball
     if (prevgs.IsBallMine && !gs.IsBallMine)
         // it is worst if we lost the ball nearby the opponent's gate
         // to make a goal opportunity
         r -= 50.0F / this.CalcDistanceToGate(gs, true);
     // if i am in position of the ball
     if (gs.IsBallMine)
         // make the agent eager to go make a goal
         r += (70.0F / this.CalcDistanceToGate(gs, true));
     else
         // make the agent eager to go and catch the ball
         r -= (30.0F * (this.CalcDistanceToBall(gs)));
     // if i made an own-goal?
     if (STACK_PEEKED && prevgs.IsBallMine && gs.Game_State == GameState.State.OPPONENT_SCORED)
         // if i made an own-goal, the direction would be 100% {WEST}.
         // if i made an own-goal, update the previous game-state's Q's value to {-INF}
         this.updateQ(prevgs, Direction.WEST, float.NegativeInfinity);
     // return the reward value
     return r;
 }
Beispiel #13
0
 /// <summary>
 /// Get valid directions of a game state
 /// </summary>
 /// <param name="gameState">The game state</param>
 /// <returns>The valid game directions</returns>
 protected IEnumerable<Direction> getValidDirections(GameState gameState)
 {
     List<Direction> directions = new List<Direction>() { /* constant direction for any state */ Direction.HOLD };
     // if not on horizontal edges?
     if (gameState.MyLocation.X > 1 && gameState.MyLocation.X < 6) directions.AddRange(new[] { Direction.NORTH, Direction.SOUTH });
     // if on top edge?
     if (gameState.MyLocation.X == 1) directions.Add(Direction.SOUTH);
     // if on bottom edge?
     if (gameState.MyLocation.X == 6) directions.Add(Direction.NORTH);
     // if not on vertical edges?
     if (gameState.MyLocation.Y > 1 && gameState.MyLocation.Y < 9) directions.AddRange(new[] { Direction.EAST, Direction.WEST });
     // if on left edge?
     if (gameState.MyLocation.Y == 1) directions.Add(Direction.EAST);
     // if on right edge?
     if (gameState.MyLocation.Y == 9) directions.Add(Direction.WEST);
     // if we are in front if any gate? /* YES: Add a direction to the gate */
     // my own gate?
     if (gameState.MyLocation.Y == 1 && (gameState.MyLocation.X == 3 || gameState.MyLocation.X == 4)) directions.Add(Direction.WEST);
     // opponent's gate?
     if (gameState.MyLocation.Y == 9 && (gameState.MyLocation.X == 3 || gameState.MyLocation.X == 4)) directions.Add(Direction.EAST);
     // return directions
     return directions;
 }