/// <summary>
/// Initializes a new instance of the <see cref="TripleModule"/> class.
/// </summary>
/// <param name="rows">The rows.</param>
/// <param name="cols">The cols.</param>
/// <param name="teammatesCount">The teammates count.</param>
/// <param name="myUnum">My unum.</param>
/// <param name="teammateToMonitor">0-based index of the teammate to monitor.</param>
/// <param name="opponentToMonitor">0-based index of the opponent to monitor.</param>
public TripleModulewIM(int rows, int cols, int teammatesCount, int opponentsCount, int myUnum, int teammateToMonitor, int opponentToMonitor)
{
m_myUnum = myUnum;
m_oppToMonitor = opponentToMonitor;
m_teammateToMonitor = teammateToMonitor;
m_teammatesCount = teammatesCount;
m_opponentsCount = opponentsCount;
m_QTable = new double[
rows, cols, // my position
rows, cols, // my teammate's position
rows, cols, // one of opponents's position
5, // ball owner index (0: me)(1: the-teammate)(2:We)(3: the opponent)(4: they)
SoccerAction.GetActionCount(Params.MoveKings, teammatesCount), // number of actions_self
SoccerAction.GetActionCount(Params.MoveKings, opponentsCount) // number of actions_opp
];
m_InternalModel = new double[
rows, cols, // my position
rows, cols, // my teammate's position
rows, cols, // one of opponents's position
5, // ball owner index (0: me)(1: the-teammate)(2:We)(3: the opponent)(4: they)
SoccerAction.GetActionCount(Params.MoveKings, opponentsCount) // number of actions_opp
];
}
public NeatCentralizedQTableBase(RLClientBase client, int numTeammates, int myUnum)
{
m_client = client;
m_rows = client.EnvRows;
m_cols = client.EnvCols;
m_numTeammates = numTeammates;
m_myUnum = myUnum;
m_numActions = SoccerAction.GetActionCount(Params.MoveKings, m_numTeammates);
this.PerformanceNetworkToEvaluate = new NeatPlayerPerformanceStats();
if (NeatExpParams.SaveFitnessGrowth)
{
m_eaLogger = new PerformanceLogger(String.Format("EALogs/{0}-{1}-{2}",
m_client.MyTeamName, m_myUnum, m_client.PerformanceLoggerMethodName), false);
m_eaLogger.WriteLine("% Generation BestFitness MeanFitness AvgComplexity");
}
Thread evoThread = new Thread(EvolutionaryThread);
evoThread.Start();
m_eventNewNetReady.WaitOne();
}
public QTable(int rows, int cols, int teammatesCount, int myUnum)
{
m_teammatesCount = teammatesCount;
m_myUnum = myUnum;
m_qTable = new double[rows, cols, rows, cols, rows, cols, rows, cols, 4,
SoccerAction.GetActionCount(Params.MoveKings, m_teammatesCount)];
}
protected virtual int RLThink()
{
int greedyActIndex = m_qTable.GetCurrentGreedyActionIndex();
int actIndex = greedyActIndex;
if (Params.Epsillon > 0.0)
{
actIndex = ChooseActionEpsilonGreedy(
Params.Epsillon,
SoccerAction.GetActionCount(Params.MoveKings, TeammatesCount),
greedyActIndex);
}
//SoccerAction act = SoccerAction.GetActionFromIndex(actIndex, Params.MoveKings, MyUnum);
if (Cycle > 0) // because in cycle 0 there is no prev state
{
double reward = EnvironmentModeler.GetReward(m_prevState, m_curState,
SoccerAction.GetActionTypeFromIndex(m_prevActionIndex, Params.MoveKings));
switch (Params.RLMethod)
{
case Params.RLMethods.Evolutionary:
m_qTable.UpdateQ_Evolutionary(m_prevActionIndex, reward);
break;
case Params.RLMethods.Q_Zero:
m_qTable.UpdateQ_QLearning(m_prevActionIndex);
break;
case Params.RLMethods.SARSA_Zero:
m_qTable.UpdateQ_SARSA(m_prevActionIndex, actIndex);
break;
case Params.RLMethods.SARSA_Lambda:
m_qTable.UpdateQ_SARSA_Lambda(m_prevActionIndex, actIndex);
break;
case Params.RLMethods.Q_Lambda_Watkins:
m_qTable.UpdateQ_QL_Watkins(m_prevActionIndex, actIndex, false);
break;
case Params.RLMethods.Q_Lambda_Naive:
m_qTable.UpdateQ_QL_Watkins(m_prevActionIndex, actIndex, true);
break;
default:
break;
}
if (m_performanceLogger.Enabled)
{
m_performanceLogger.Log(Cycle, reward, OurScore, OppScore);
}
}
return(actIndex);
}
private SoccerAction RandomPlayer()
{
int[] teammates = this.GetAvailableTeammatesUnums().ToArray();
int numActions = SoccerAction.GetActionCount(this.MoveKings, teammates.Length + 1);
int ai = rnd.Next(0, numActions);
return(SoccerAction.GetActionFromIndex(ai, this.MoveKings, this.MyUnum));
}
protected override double GetQValue(State s, int ai_self)
{
int a_other_count = SoccerAction.GetActionCount(Params.MoveKings, m_opponentsCount);
double sum = 0.0;
for (int i = 0; i < a_other_count; ++i)
{
sum += GetQValue(s, ai_self, i) * GetIMValue(s, i);
}
return(sum);
}
public SelfOnlyModule(int rows, int cols, int teammatesCount, int myUnum)
{
m_myUnum = myUnum;
m_teammatesCount = teammatesCount;
m_QTable = new double[
rows, cols, // my position
3, // ball owner status (0: Me)(1: We)(2: Opp) own the ball
SoccerAction.GetActionCount(Params.MoveKings, teammatesCount) // number of actions
];
}
public virtual int GetStateCount(State s)
{
int numActions = SoccerAction.GetActionCount(Params.MoveKings, TeammatesCount);
int sum = 0;
for (int i = 0; i < numActions; ++i)
{
sum += GetCountValue(s, i);
}
return(sum);
}
/// <summary>
/// Initializes a new instance of the <see cref="SelfAndTeammateModule"/> class.
/// </summary>
/// <param name="rows">The rows.</param>
/// <param name="cols">The cols.</param>
/// <param name="teammatesCount">The teammates count.</param>
/// <param name="myUnum">My unum.</param>
/// <param name="teammateToMonitor">The 0-based index of the teammate to monitor.</param>
public SelfAndTeammateModule(int rows, int cols, int teammatesCount, int myUnum, int teammateToMonitor)
{
m_myUnum = myUnum;
m_teammatesCount = teammatesCount;
m_teammateToMonitor = teammateToMonitor;
m_QTable = new double[
rows, cols, // my position
rows, cols, // my teammate position
4, // ball owner status (0: Me)(1: The Teammate)(2: We)(3: Opp) own the ball
SoccerAction.GetActionCount(Params.MoveKings, teammatesCount) // number of actions
];
}
/// <summary>
/// Initializes a new instance of the <see cref="SelfAndOneOpponentModule"/> class.
/// </summary>
/// <param name="rows">The rows.</param>
/// <param name="cols">The cols.</param>
/// <param name="teammatesCount">The teammates count.</param>
/// <param name="myUnum">My unum.</param>
/// <param name="opponentToMonitor">0-based index of the opponent to monitor.</param>
public SelfAndOneOpponentModule(int rows, int cols, int teammatesCount, int myUnum, int opponentToMonitor)
{
m_myUnum = myUnum;
m_oppToMonitor = opponentToMonitor;
m_teammatesCount = teammatesCount;
m_QTable = new double[
rows, cols, // my position
rows, cols, // one of opponents's position
4, // ball owner player's index (0: Me) (1: We)(2: The Opponent) (3: They)
SoccerAction.GetActionCount(Params.MoveKings, teammatesCount) // number of actions
];
}
public SelfOnlyCounterModule(int rows, int cols, int teammatesCount, int myUnum)
{
m_rows = rows;
m_cols = cols;
m_actionsCount = SoccerAction.GetActionCount(Params.MoveKings, teammatesCount);
m_myUnum = myUnum;
m_ballOwnerStatesCount = 3; // ball owner status (0: Me)(1: We)(2: Opp) own the ball
m_teammatesCount = teammatesCount;
m_counterTable = new int[
rows, cols, // my position
m_ballOwnerStatesCount, // ball owner status (0: Me)(1: We)(2: Opp) own the ball
m_actionsCount // number of actions
];
}
private double GetBestQ(State s, int ai_prime_other)
{
int a_self_count = SoccerAction.GetActionCount(Params.MoveKings, m_teammatesCount);
double maxQ = Double.MinValue;
double curQ;
for (int i = 0; i < a_self_count; ++i)
{
curQ = GetQValue(s, i, ai_prime_other);
if (curQ > maxQ)
{
maxQ = curQ;
}
}
return(maxQ);
}
private void UpdateIM_Learning(State s, int ai_star_other)
{
int a_other_count = SoccerAction.GetActionCount(Params.MoveKings, m_opponentsCount);
double curIMValue, newIMValue;
for (int i = 0; i < a_other_count; ++i)
{
curIMValue = GetIMValue(s, i);
newIMValue = (1 - Params.Theta) * curIMValue;
if (i == ai_star_other)
{
newIMValue += Params.Theta;
}
UpdateIMValue(s, i, newIMValue);
}
}
protected int GetMaxIMIndex(State s)
{
int a_other_count = SoccerAction.GetActionCount(Params.MoveKings, m_opponentsCount);
int maxIndex = 0;
double maxValue = Double.MinValue;
double curValue;
for (int i = 0; i < a_other_count; ++i)
{
curValue = GetIMValue(s, i);
if (curValue > maxValue)
{
maxValue = curValue;
maxIndex = i;
}
}
return(maxIndex);
}
/// <summary>
/// Initializes a new instance of the <see cref="SelfAndTeammateModule"/> class.
/// </summary>
/// <param name="rows">The rows.</param>
/// <param name="cols">The cols.</param>
/// <param name="teammatesCount">The teammates count.</param>
/// <param name="myUnum">My unum.</param>
/// <param name="teammateToMonitor">The 0-based index of the teammate to monitor.</param>
public SelfAndTeammateCounterModule(int rows, int cols, int teammatesCount, int myUnum, int teammateToMonitor)
{
m_rows = rows;
m_cols = cols;
m_actionsCount = SoccerAction.GetActionCount(Params.MoveKings, teammatesCount);
m_ballOwnerStatesCount = 4; // ball owner status (0: Me)(1: The Teammate)(2: We)(3: Opp) own the ball
m_myUnum = myUnum;
m_teammatesCount = teammatesCount;
m_teammateToMonitor = teammateToMonitor;
m_counterTable = new int[
rows, cols, // my position
rows, cols, // my teammate position
m_ballOwnerStatesCount, // ball owner status (0: Me)(1: The Teammate)(2: We)(3: Opp) own the ball
m_actionsCount // number of actions
];
}
/// <summary>
/// Initializes a new instance of the <see cref="TripleModule"/> class.
/// </summary>
/// <param name="rows">The rows.</param>
/// <param name="cols">The cols.</param>
/// <param name="teammatesCount">The teammates count.</param>
/// <param name="myUnum">My unum.</param>
/// <param name="teammateToMonitor">0-based index of the teammate to monitor.</param>
/// <param name="opponentToMonitor">0-based index of the opponent to monitor.</param>
public PartialSelfAndOneOpponentModule(int rows, int cols, int dist, int teammatesCount, int oppsCount, int myUnum)
{
m_myUnum = myUnum;
m_teammatesCount = teammatesCount;
m_oppsCount = oppsCount;
m_dist = dist;
int playerStates = (2 * m_dist + 1) * (2 * m_dist + 1) + 1;
m_QTable = new double[
playerStates, // my teammate's position
playerStates, // one of opponents's position
playerStates, // the other opponents's position
5, // ball owner index (0: me)(1: the-teammate)(2:Opp1)(3: Opp2)(4: Unknown)
SoccerAction.GetActionCount(Params.MoveKings, teammatesCount) // number of actions
];
}
/// <summary>
/// Initializes a new instance of the <see cref="SelfAndOneOpponentModule"/> class.
/// </summary>
/// <param name="rows">The rows.</param>
/// <param name="cols">The cols.</param>
/// <param name="teammatesCount">The teammates count.</param>
/// <param name="myUnum">My unum.</param>
/// <param name="opponentToMonitor">0-based index of the opponent to monitor.</param>
public SelfAndOneOpponentCounterModule(int rows, int cols, int teammatesCount, int myUnum, int opponentToMonitor)
{
m_rows = rows;
m_cols = cols;
m_actionsCount = SoccerAction.GetActionCount(Params.MoveKings, teammatesCount);
m_ballOwnerStatesCount = 4; // ball owner player's index (0: Me) (1: We)(2: The Opponent) (3: They)
m_myUnum = myUnum;
m_oppToMonitor = opponentToMonitor;
m_teammatesCount = teammatesCount;
m_counterTable = new int[
rows, cols, // my position
rows, cols, // one of opponents's position
m_ballOwnerStatesCount, // ball owner player's index (0: Me) (1: We)(2: The Opponent) (3: They)
m_actionsCount // number of actions
];
}
/// <summary>
/// Initializes a new instance of the <see cref="TripleModule"/> class.
/// </summary>
/// <param name="rows">The rows.</param>
/// <param name="cols">The cols.</param>
/// <param name="teammatesCount">The teammates count.</param>
/// <param name="myUnum">My unum.</param>
/// <param name="teammateToMonitor">0-based index of the teammate to monitor.</param>
/// <param name="opponentToMonitor">0-based index of the opponent to monitor.</param>
public TripleCounterModule(int rows, int cols, int teammatesCount, int myUnum, int teammateToMonitor, int opponentToMonitor)
{
m_rows = rows;
m_cols = cols;
m_actionsCount = SoccerAction.GetActionCount(Params.MoveKings, teammatesCount);
m_ballOwnerStatesCount = 5; // ball owner index (0: me)(1: the-teammate)(2:We)(3: the opponent)(4: they)
m_myUnum = myUnum;
m_oppToMonitor = opponentToMonitor;
m_teammateToMonitor = teammateToMonitor;
m_teammatesCount = teammatesCount;
m_counterTable = new int[
rows, cols, // my position
rows, cols, // my teammate's position
rows, cols, // one of opponents's position
m_ballOwnerStatesCount, // ball owner index (0: me)(1: the-teammate)(2:We)(3: the opponent)(4: they)
m_actionsCount // number of actions
];
}
public int GetMaxQ(State s, out double maxQValue)
{
double curValue = 0.0;
maxQValue = Double.MinValue;
int maxIndex = 0;
int count = SoccerAction.GetActionCount(Params.MoveKings, TeammatesCount);
for (int i = 0; i < count; ++i)
{
curValue = GetQValue(s, i);
if (curValue > maxQValue)
{
maxQValue = curValue;
maxIndex = i;
}
}
return(maxIndex);
}
/// <summary>
/// Initializes a new instance of the <see cref="TripleModule"/> class.
/// </summary>
/// <param name="rows">The rows.</param>
/// <param name="cols">The cols.</param>
/// <param name="teammatesCount">The teammates count.</param>
/// <param name="myUnum">My unum.</param>
/// <param name="teammateToMonitor">0-based index of the teammate to monitor.</param>
/// <param name="opponentToMonitor">0-based index of the opponent to monitor.</param>
public PartialCounterModule(int rows, int cols, int dist, int teammatesCount, int oppsCount, int myUnum)
{
m_rows = rows;
m_cols = cols;
m_actionsCount = SoccerAction.GetActionCount(Params.MoveKings, teammatesCount);
m_ballOwnerStatesCount = 5; // ball owner index (0: me)(1: the-teammate)(2:Opp1)(3: Opp2)(4: Unknown)
m_myUnum = myUnum;
m_teammatesCount = teammatesCount;
m_oppsCount = oppsCount;
m_dist = dist;
m_playerStates = (2 * m_dist + 1) * (2 * m_dist + 1) + 1;
m_counterTable = new int[
m_playerStates, // my teammate's position
m_playerStates, // one of opponents's position
m_playerStates, // the other opponents's position
m_ballOwnerStatesCount, // ball owner index (0: me)(1: the-teammate)(2:Opp1)(3: Opp2)(4: Unknown)
m_actionsCount // number of actions
];
}
public HyperNEATQTable(RLClientBase client, int numTeammates, int myUnum)
{
m_client = client;
m_rows = client.EnvRows;
m_cols = client.EnvCols;
m_numTeammates = numTeammates;
m_myUnum = myUnum;
m_numActions = SoccerAction.GetActionCount(Params.MoveKings, m_numTeammates);
if (NeatExpParams.SaveFitnessGrowth)
{
m_eaLogger = new PerformanceLogger(String.Format("EALogs/{0}-{1}",
m_client.MyTeamName, m_myUnum), false);
m_eaLogger.WriteLine("% generation bestfitness");
}
Thread evoThread = new Thread(EvolutionaryThread);
evoThread.Start();
m_eventNewNetReady.WaitOne();
}
protected override SoccerAction Think()
{
int[] teammates = this.GetAvailableTeammatesUnums().ToArray();
int numActions = SoccerAction.GetActionCount(this.MoveKings, teammates.Length + 1);
int ai = rnd.Next(0, numActions);
return(SoccerAction.GetActionFromIndex(ai, this.MoveKings, this.MyUnum));
//int dst = -1;
//ActionTypes at = ActionTypes.Hold;
//SoccerAction.GetActionTypeFromIndex(ai);
//at = (ActionTypes)
//if (at == ActionTypes.Pass)
//{
// if (teammates.Length <= 0)
// at = ActionTypes.Hold;
// else
// dst = teammates[rnd.Next(0, teammates.Length)];
//}
//return new SoccerAction(at, dst);
}
public RL1PQTable(int rows, int cols, int unum)
{
m_qTable = new double[rows, cols, rows, cols, 2,
SoccerAction.GetActionCount(Params.MoveKings, TeammatesCount)];
m_myUnum = unum;
}
protected void UpdateQ_SARSA_Lambda(double reward, State prevState, State curState, int prevActIndex, int curActIndex)
{
double oldQ = GetQValue(prevState, prevActIndex);
double qOfNewState = GetQValue(curState, curActIndex);
double delta;
if (m_updateOnlyTDTarget)
{
delta = reward + Params.Gamma * qOfNewState;
}
else
{
delta = reward + Params.Gamma * qOfNewState - oldQ;
}
if (!m_eTrace.ContainsStateActionPair(prevState, prevActIndex))
{
m_eTrace.AddStateActionPair(prevState, prevActIndex, 0.0);
}
else
{
Console.WriteLine("Trace exists! " + prevState.GetStateString());
}
if (Params.TraceType == Params.EligibilityTraceTypes.Accumulating)
{
m_eTrace.IncrementValue(prevState, prevActIndex);
}
else if (Params.TraceType == Params.EligibilityTraceTypes.Replacing)
{
m_eTrace.RemoveTracesForState(prevState, prevActIndex, SoccerAction.GetActionCount(Params.MoveKings, TeammatesCount));
m_eTrace.UpdateValue(prevState, prevActIndex, 1.0);
}
var listTraceItems = m_eTrace.GetTraceItems().ToList();
foreach (var pair in listTraceItems)
{
double q = GetQValue(pair.Key.State, pair.Key.ActionIndex);
double e = pair.Value;
double newQ;
if (m_updateOnlyTDTarget)
{
newQ = delta * e;
}
else
{
newQ = q + Params.Alpha * delta * e;
}
UpdateQValue(pair.Key.State, pair.Key.ActionIndex, newQ);
m_eTrace.MultiplyValue(pair.Key.State, pair.Key.ActionIndex, Params.Gamma * Params.Lambda);
}
// see if episode changed
if (curState.OurScore != prevState.OurScore || curState.OppScore != prevState.OppScore)
{
m_eTrace.ClearTrace();
}
}