/// <summary> /// Returns the maximum Q-value found for any moves performable in the given state. /// If there is no data for a move, it will be considered having [initvalue]. /// If there is no data for the state, the return value will be 0. /// </summary> /// <param name="state"></param> /// <returns></returns> private double GetMaxQ(VirusBoard state) { if (state.Equals(default(VirusBoard)) || !Q.ContainsKey(state.CustomHash())) { return(0); } double max = -10; Move[] actions = state.GetPossibleMoves(playerNumber); foreach (Move a in actions) { double value = 0; if (!Q[state.CustomHash()].ContainsKey(a.CustomHash())) { value = initvalue; } else { value = Q[state.CustomHash()][a.CustomHash()]; } if (value > max) { max = value; } } return(max); }
private Move GetMaxExplorationFunctionA(VirusBoard state) { double max = double.NegativeInfinity; Move action = default(Move); Move[] actions = state.GetPossibleMoves(playerNumber); if (!Q.ContainsKey(state.CustomHash())) { return(actions.Length > 0 ? actions[0] : action); } bool berandom = random.NextDouble() < RandomRate; foreach (Move a in actions) { double value = 0; if (Q[state.CustomHash()].ContainsKey(a.CustomHash())) { if (Q[state.CustomHash()][a.CustomHash()] >= 1) { value = 1; max = value; action = a; break; } else if (Q[state.CustomHash()][a.CustomHash()] <= -1) { value = -1; } else { if (berandom) { value = random.NextDouble(); } else { value = Q[state.CustomHash()][a.CustomHash()] + ((explore && !(RandomRate > 0)) ? ExplorationRate(N[state.CustomHash()][a.CustomHash()]) : 0); } } } else { value = 1; } if (value > max) { max = value; action = a; } } return(action); }
/// <summary> /// Returns the maximum Q-value found for any moves performable in the given state. /// If there is no data for a move, it will be considered having [initvalue]. /// If there is no data for the state, the return value will be 0. /// </summary> /// <param name="state"></param> /// <returns></returns> private double GetMaxQ(VirusBoard state) { if (state.Equals(default(VirusBoard)) || !Q.ContainsKey(state.CustomHash())) return 0; double max = -10; Move[] actions = state.GetPossibleMoves(playerNumber); foreach (Move a in actions) { double value = 0; if (!Q[state.CustomHash()].ContainsKey(a.CustomHash())) { value = initvalue; } else { value = Q[state.CustomHash()][a.CustomHash()]; } if (value > max) max = value; } return max; }
//Calc maxValue private double MaxValue(VirusBoard state, int iteration) { iteration++; if (state.winner == playerNumber) { return double.PositiveInfinity; } if (state.winner != playerNumber && state.winner != 0) { return double.NegativeInfinity; } if (iteration < searchLength) { Move[] actions = state.GetPossibleMoves(playerNumber); double max = double.NegativeInfinity; foreach (Move a in actions) { VirusBoard newState = state.GetUpdated(a); double q = Utility(state, newState); if (Q.ContainsKey(state.CustomHash())) { if (Q[state.CustomHash()].ContainsKey(a.CustomHash())) { q = Q[state.CustomHash()][a.CustomHash()]; } } q += MinValue(newState, iteration); if (q > max) { max = q; } if (max == double.PositiveInfinity) { return max; } } return max; } else { return 0; } }
private Move GetMaxExplorationFunctionA(VirusBoard state) { double max = double.NegativeInfinity; Move action = default(Move); Move[] actions = state.GetPossibleMoves(playerNumber); if (!Q.ContainsKey(state.CustomHash())) return actions.Length > 0 ? actions[0] : action; bool berandom = random.NextDouble() < RandomRate; foreach (Move a in actions) { double value = 0; if (Q[state.CustomHash()].ContainsKey(a.CustomHash())) { if (Q[state.CustomHash()][a.CustomHash()] >= 1) { value = 1; max = value; action = a; break; } else if (Q[state.CustomHash()][a.CustomHash()] <= -1) { value = -1; } else { if (berandom) value = random.NextDouble(); else value = Q[state.CustomHash()][a.CustomHash()] + ((explore && !(RandomRate > 0)) ? ExplorationRate(N[state.CustomHash()][a.CustomHash()]) : 0); } } else { value = 1; } if (value > max) { max = value; action = a; } } return action; }
// Calc minValue private double MinValue(VirusBoard state, int iteration) { iteration++; if (state.winner == playerNumber) { return double.PositiveInfinity; } if (state.winner != playerNumber && state.winner != 0) { return double.NegativeInfinity; } if (iteration < searchLength) { byte opponent = (playerNumber == 1) ? (byte)2 : (byte)1; Move[] actions = state.GetPossibleMoves(opponent); double min = double.PositiveInfinity; foreach (Move a in actions) { VirusBoard newState = state.GetUpdated(a); double q = Utility(state, newState); if (Q.ContainsKey(state.CustomHash())) { if (Q[state.CustomHash()].ContainsKey(a.CustomHash())) { q = -Q[state.CustomHash()][a.CustomHash()]; } } q += MaxValue(newState, iteration); if (q < min) { min = q; } if (min == double.NegativeInfinity) { return min; } } return min; } else { return 0; } }
private Move GetMaxExplorationFunctionA(VirusBoard state) { double max = double.NegativeInfinity; Move action = default(Move); Move[] actions = state.GetPossibleMoves(playerNumber); bool berandom = random.NextDouble() < RandomRate; foreach (Move a in actions) { double value = 0; if (!Q.ContainsKey(state.CustomHash())) { Q.Add(state.CustomHash(), new Dictionary<UInt32, double>()); } if (Q[state.CustomHash()].ContainsKey(a.CustomHash())) { if (Q[state.CustomHash()][a.CustomHash()] >= 1) { value = 1; max = value; action = a; break; } else { if (berandom) value = random.NextDouble(); else value = Q[state.CustomHash()][a.CustomHash()] + (explore ? ExplorationRate(N[state.CustomHash()][a.CustomHash()]) : 0); } } else { value = 1; } if (value > max) { max = value; action = a; } } return action; }
private double GetMaxQ(VirusBoard state) { double max = -10; Move[] actions = state.GetPossibleMoves(playerNumber); foreach (Move a in actions) { double value = 0; if (!Q[state.CustomHash()].ContainsKey(a.CustomHash())) { value = 0; } else { value = Q[state.CustomHash()][a.CustomHash()]; } if (value > max) max = value; } if (Q[state.CustomHash()].ContainsKey(0)) { if (Q[state.CustomHash()][0] > max) max = Q[state.CustomHash()][0]; } return max; }
public void Learn(VirusBoard startstate, VirusBoard endstate, Move action, double reward) { // -- Make sure the entries for the state and action exist -- if (!Q.ContainsKey(startstate.CustomHash())) Q.Add(startstate.CustomHash(), new Dictionary<UInt32, double>()); if (!Q[startstate.CustomHash()].ContainsKey(action.CustomHash())) Q[startstate.CustomHash()].Add(action.CustomHash(), initvalue); if (!N.ContainsKey(startstate.CustomHash())) N.Add(startstate.CustomHash(), new Dictionary<UInt32, int>()); if (!N[startstate.CustomHash()].ContainsKey(action.CustomHash())) N[startstate.CustomHash()].Add(action.CustomHash(), 0); // -- Perform the update of Q-values -- N[startstate.CustomHash()][action.CustomHash()]++; Q[startstate.CustomHash()][action.CustomHash()] = Q[startstate.CustomHash()][action.CustomHash()] + LearningRate(N[startstate.CustomHash()][action.CustomHash()]) * (reward + discount * GetMaxQ(endstate) - Q[startstate.CustomHash()][action.CustomHash()]); }
public override Move Move(Virus percept) { VirusBoard newState = percept.GetBoardCopy(); if (!Q.ContainsKey(newState.CustomHash())) Q.Add(newState.CustomHash(), new Dictionary<UInt32, double>()); if (learn && !prevState.Equals(default(VirusBoard))) { if (!N.ContainsKey(prevState.CustomHash())) N.Add(prevState.CustomHash(), new Dictionary<UInt32, int>()); if (!N[prevState.CustomHash()].ContainsKey(prevAction.CustomHash())) N[prevState.CustomHash()].Add(prevAction.CustomHash(), 0); N[prevState.CustomHash()][prevAction.CustomHash()]++; Q[prevState.CustomHash()][prevAction.CustomHash()] = Q[prevState.CustomHash()][prevAction.CustomHash()] + LearningRate(N[prevState.CustomHash()][prevAction.CustomHash()]) * (prevReward + discount * GetMaxQ(newState) - Q[prevState.CustomHash()][prevAction.CustomHash()]); } prevState = newState; prevAction = GetMaxExplorationFunctionA(newState); prevReward = 0; if (learn && !Q[prevState.CustomHash()].ContainsKey(prevAction.CustomHash())) Q[prevState.CustomHash()].Add(prevAction.CustomHash(), initvalue); return prevAction; }
private double Learn(VirusMemory memory) { VirusBoard startstate = memory.StartState; VirusBoard endstate = memory.EndState; Move action = memory.Action; double reward = memory.Reward; // -- Make sure the entries for the state and action exist -- if (!Q.ContainsKey(startstate.CustomHash())) { Q.Add(startstate.CustomHash(), new Dictionary <UInt32, double>()); } if (!Q[startstate.CustomHash()].ContainsKey(action.CustomHash())) { Q[startstate.CustomHash()].Add(action.CustomHash(), initvalue); } if (!N.ContainsKey(startstate.CustomHash())) { N.Add(startstate.CustomHash(), new Dictionary <UInt32, int>()); } if (!N[startstate.CustomHash()].ContainsKey(action.CustomHash())) { N[startstate.CustomHash()].Add(action.CustomHash(), 0); } // -- Perform the update of Q-values -- N[startstate.CustomHash()][action.CustomHash()]++; double change = LearningRate(N[startstate.CustomHash()][action.CustomHash()]) * (reward + discount * GetMaxQ(endstate) - Q[startstate.CustomHash()][action.CustomHash()]); Q[startstate.CustomHash()][action.CustomHash()] = Q[startstate.CustomHash()][action.CustomHash()] + change; return(change); }
public void Learn(VirusBoard startstate, VirusBoard endstate, Move action, double reward) { // -- Make sure the entries for the state and action exist -- if (!Q.ContainsKey(startstate.CustomHash())) { Q.Add(startstate.CustomHash(), new Dictionary <UInt32, double>()); } if (!Q[startstate.CustomHash()].ContainsKey(action.CustomHash())) { Q[startstate.CustomHash()].Add(action.CustomHash(), initvalue); } if (!N.ContainsKey(startstate.CustomHash())) { N.Add(startstate.CustomHash(), new Dictionary <UInt32, int>()); } if (!N[startstate.CustomHash()].ContainsKey(action.CustomHash())) { N[startstate.CustomHash()].Add(action.CustomHash(), 0); } // -- Perform the update of Q-values -- N[startstate.CustomHash()][action.CustomHash()]++; Q[startstate.CustomHash()][action.CustomHash()] = Q[startstate.CustomHash()][action.CustomHash()] + LearningRate(N[startstate.CustomHash()][action.CustomHash()]) * (reward + discount * GetMaxQ(endstate) - Q[startstate.CustomHash()][action.CustomHash()]); }