private static void RunFoeQ() { var S = new StateSet(); var A = new JointActionSet(); var P = new Transition(S, A); var Q_A = new FoeQTable(); var R = new Rewards(); var ERR = new Dictionary <int, double>(); var j = 0; while (j <= 1000000) { //Initialize state according to Figure 4 var initialState = new State(2, 1, BallPossessor.B); var done = false; var currState = new State(3, 1, BallPossessor.B); var actions = A.GetNextJointAction(); var playerAAction = actions.currplayer1Act; var playerBAction = actions.player2act; while (!done) { if (j % 20000 == 0) { Console.WriteLine(j); } var q_fig_4_initial = Q_A.getQval(initialState, Action.S, Action.X); var nextState = P.GetNextState(currState, new JointAction(playerAAction, playerBAction)); var playerAReward = R.GetPlayerAReward(nextState); var playerBReward = R.GetPlayerBReward(nextState); if (playerAReward != 0.0 || playerBReward != 0.0) { done = true; } Q_A.UpdateQValue(currState, nextState, playerAAction, playerBAction, playerAReward); if (currState.Equals(initialState) && playerAAction == Action.S && playerBAction == Action.X) { var q_fig_4 = Q_A.getQval(initialState, Action.S, Action.X); var diff = Math.Abs(q_fig_4 - q_fig_4_initial); ERR.Add(j, diff); } currState = nextState; actions = A.GetNextJointAction(); playerAAction = actions.currplayer1Act; playerBAction = actions.player2act; ++j; } } using (StreamWriter sw = File.CreateText("output.csv")) { foreach (var kvp in ERR) { sw.WriteLine(kvp.Key + "," + kvp.Value); } } }
private static void RunCorrelatedQ() { var S = new StateSet(); var A = new JointActionSet(); var P = new Transition(S, A); var Q_Joint = new CorrelatedQTable(); var R = new Rewards(); var ERR = new Dictionary <int, double>(); var j = 0; while (j <= 1000000) { //Initialize state according to Figure 4 var initialState = new State(2, 1, BallPossessor.B); var gameOver = false; var currentState = initialState; var actions = A.GetNextJointAction(); var playerAAction = actions.CurrentPlayerAction; var playerBAction = actions.OpposingPlayerAction; while (!gameOver) { if (j % 20000 == 0) { Console.WriteLine(j); } var q_fig_4_initial = Q_Joint.GetCurrentPlayerQValue(initialState, Action.South, Action.Stick); var nextState = P.GetNextState(currentState, new JointAction(playerAAction, playerBAction)); var playerAReward = R.GetPlayerAReward(nextState); var playerBReward = R.GetPlayerBReward(nextState); if (playerAReward != 0.0 || playerBReward != 0.0) { gameOver = true; } Q_Joint.UpdateQValue(currentState, nextState, playerAAction, playerBAction, playerAReward, playerBReward, gameOver); if (currentState.Equals(initialState) && playerAAction == Action.South && playerBAction == Action.Stick) { var q_fig_4 = Q_Joint.GetCurrentPlayerQValue(initialState, Action.South, Action.Stick); var diff = Math.Abs(q_fig_4 - q_fig_4_initial); Console.WriteLine(diff); ERR.Add(j, diff); } currentState = nextState; actions = A.GetNextJointAction(); playerAAction = actions.CurrentPlayerAction; playerBAction = actions.OpposingPlayerAction; ++j; } } using (StreamWriter sw = File.CreateText("output.csv")) { foreach (var kvp in ERR) { sw.WriteLine(kvp.Key + "," + kvp.Value); } } }