private static void RunFoeQ()
        {
            var S   = new StateSet();
            var A   = new JointActionSet();
            var P   = new Transition(S, A);
            var Q_A = new FoeQTable();
            var R   = new Rewards();

            var ERR = new Dictionary <int, double>();

            var j = 0;

            while (j <= 1000000)
            {
                //Initialize state according to Figure 4
                var initialState = new State(2, 1, BallPossessor.B);
                var done         = false;
                var currState    = new State(3, 1, BallPossessor.B);

                var actions = A.GetNextJointAction();

                var playerAAction = actions.currplayer1Act;
                var playerBAction = actions.player2act;

                while (!done)
                {
                    if (j % 20000 == 0)
                    {
                        Console.WriteLine(j);
                    }

                    var q_fig_4_initial = Q_A.getQval(initialState, Action.S, Action.X);

                    var nextState = P.GetNextState(currState, new JointAction(playerAAction, playerBAction));

                    var playerAReward = R.GetPlayerAReward(nextState);
                    var playerBReward = R.GetPlayerBReward(nextState);

                    if (playerAReward != 0.0 || playerBReward != 0.0)
                    {
                        done = true;
                    }

                    Q_A.UpdateQValue(currState, nextState, playerAAction, playerBAction, playerAReward);

                    if (currState.Equals(initialState) && playerAAction == Action.S && playerBAction == Action.X)
                    {
                        var q_fig_4 = Q_A.getQval(initialState, Action.S, Action.X);
                        var diff    = Math.Abs(q_fig_4 - q_fig_4_initial);

                        ERR.Add(j, diff);
                    }

                    currState = nextState;

                    actions = A.GetNextJointAction();

                    playerAAction = actions.currplayer1Act;
                    playerBAction = actions.player2act;

                    ++j;
                }
            }

            using (StreamWriter sw = File.CreateText("output.csv"))
            {
                foreach (var kvp in ERR)
                {
                    sw.WriteLine(kvp.Key + "," + kvp.Value);
                }
            }
        }
        private static void RunCorrelatedQ()
        {
            var S       = new StateSet();
            var A       = new JointActionSet();
            var P       = new Transition(S, A);
            var Q_Joint = new CorrelatedQTable();
            var R       = new Rewards();

            var ERR = new Dictionary <int, double>();

            var j = 0;

            while (j <= 1000000)
            {
                //Initialize state according to Figure 4
                var initialState = new State(2, 1, BallPossessor.B);
                var gameOver     = false;
                var currentState = initialState;

                var actions = A.GetNextJointAction();

                var playerAAction = actions.CurrentPlayerAction;
                var playerBAction = actions.OpposingPlayerAction;

                while (!gameOver)
                {
                    if (j % 20000 == 0)
                    {
                        Console.WriteLine(j);
                    }

                    var q_fig_4_initial = Q_Joint.GetCurrentPlayerQValue(initialState, Action.South, Action.Stick);

                    var nextState = P.GetNextState(currentState, new JointAction(playerAAction, playerBAction));

                    var playerAReward = R.GetPlayerAReward(nextState);
                    var playerBReward = R.GetPlayerBReward(nextState);

                    if (playerAReward != 0.0 || playerBReward != 0.0)
                    {
                        gameOver = true;
                    }

                    Q_Joint.UpdateQValue(currentState, nextState, playerAAction, playerBAction, playerAReward, playerBReward, gameOver);

                    if (currentState.Equals(initialState) && playerAAction == Action.South && playerBAction == Action.Stick)
                    {
                        var q_fig_4 = Q_Joint.GetCurrentPlayerQValue(initialState, Action.South, Action.Stick);
                        var diff    = Math.Abs(q_fig_4 - q_fig_4_initial);
                        Console.WriteLine(diff);
                        ERR.Add(j, diff);
                    }

                    currentState = nextState;

                    actions = A.GetNextJointAction();

                    playerAAction = actions.CurrentPlayerAction;
                    playerBAction = actions.OpposingPlayerAction;

                    ++j;
                }
            }

            using (StreamWriter sw = File.CreateText("output.csv"))
            {
                foreach (var kvp in ERR)
                {
                    sw.WriteLine(kvp.Key + "," + kvp.Value);
                }
            }
        }