public override void UpdateMemoryBlocks()
        {
            UtilityOutput.Count = NoActions;
            RewardStats.Count   = 2;

            LearningParams    = new MyModuleParams();
            Memory            = new MyQSAMemory(GlobalDataInput.Count, NoActions);
            LearningAlgorithm = new MyDiscreteQLearning(LearningParams, Memory);

            if (GlobalDataInput != null)
            {
                if (NoActions == 6)
                {
                    MyLog.DEBUG.WriteLine("6 actions set by the user, will use action names for gridworld");
                    Rds = new MyRootDecisionSpace(GlobalDataInput.Count, new String[] { " -", " <", " >", " ^", " v", " P" }, LearningParams);
                }
                else if (NoActions == 3)
                {
                    MyLog.DEBUG.WriteLine("3 actions set by the user, will use action names for pong");
                    Rds = new MyRootDecisionSpace(GlobalDataInput.Count, new String[] { " <", " -", " >" }, LearningParams);
                }
                else
                {
                    MyLog.DEBUG.WriteLine("Unknown no. of actions, will use automatic naming of actions");
                    String[] names = new String[NoActions];
                    for (int i = 0; i < NoActions; i++)
                    {
                        names[i] = "A" + i;
                    }
                    Rds = new MyRootDecisionSpace(GlobalDataInput.Count, names, LearningParams);
                }
                CurrentStateOutput.Count = GlobalDataInput.Count;
            }
        }
示例#2
0
 public MyActionHistory(ActionManager am, MyModuleParams setup)
 {
     this.setup       = setup;
     this.am          = am;
     this.changed     = new List <int>();
     this.isMonitored = new Dictionary <int, bool>();
 }
        public override void UpdateMemoryBlocks()
        {
            UtilityOutput.Count = NoActions;
            LearningParams      = new MyModuleParams();

            if (GlobalDataInput != null)
            {
                if (NoActions == 6)
                {
                    MyLog.DEBUG.WriteLine("6 actions set by the user, will use action names for gridworld");
                    Rds = new MyRootDecisionSpace(GlobalDataInput.Count, new String[] { " -", " <", " >", " ^", " v", " P" }, LearningParams);
                }
                else if (NoActions == 3)
                {
                    MyLog.DEBUG.WriteLine("3 actions set by the user, will use action names for pong");
                    Rds = new MyRootDecisionSpace(GlobalDataInput.Count, new String[] { " <", " -", " >" }, LearningParams);
                }
                else
                {
                    MyLog.DEBUG.WriteLine("Unknown no. of actions, will use automatic naming of actions");
                    String[] names = new String[NoActions];
                    for (int i = 0; i < NoActions; i++)
                    {
                        names[i] = "A" + i;
                    }
                    Rds = new MyRootDecisionSpace(GlobalDataInput.Count, names, LearningParams);
                }
                Hierarchy                = new MyHierarchyMaintainer(Rds, LearningParams);
                SubspaceUtils.Count      = MaxActions * MaxActions;
                SubspaceUtils.ColumnHint = MaxActions;
            }
        }
 public MyActionHistory(ActionManager am, MyModuleParams setup)
 {
     this.setup = setup;
     this.am = am;
     this.changed = new List<int>();
     this.isMonitored = new Dictionary<int, bool>();
 }
示例#5
0
        public MyDiscreteQLearning(MyModuleParams learningParams, MyQSAMemory mem)
        {
            this.m_mem      = mem;
            this.m_learning = learningParams;

            m_trace = new MyEligibilityTrace(learningParams);
            m_trace.PushState(new int[mem.GetMaxStateVariables()]);
        }
        public MyDiscreteQLearning(MyModuleParams learningParams, MyQSAMemory mem)
        {
            this.m_mem = mem;
            this.m_learning = learningParams;

            m_trace = new MyEligibilityTrace(learningParams);
            m_trace.PushState(new int[mem.GetMaxStateVariables()]);
        }
示例#7
0
        public MyVariableHistory(VariableManager vm, MyModuleParams setup)
        {
            this.vm    = vm;
            this.setup = setup;

            this.changed = new List <List <int> >();

            isMonitored = new bool[vm.GetMaxVariables()];
        }
        public MyDecisionSpace(MyStochasticReturnPredictor mySRP, MyRootDecisionSpace rds, MyModuleParams setup)
        {
            this.m_setup = setup;
            this.m_rds   = rds;
            this.m_mySRP = mySRP;
            m_S_t        = new int[rds.VarManager.MAX_VARIABLES];

            ChildActions     = new MyActionLink(0, rds);
            m_childVariables = new MyVariableLink(rds.VarManager.GetMaxVariables());
        }
 public ActionManager(String[] labels, MyModuleParams setup)
 {
     this.m_setup = setup;
     Actions = new List<MyMotivatedAction>();
     for (int i = 0; i < labels.Length; i++)
     {
         Actions.Add(new MyMotivatedAction(labels[i], 0, setup));
     }
     this.AcitonHistory = new MyActionHistory(this, setup);
     this.AcitonHistory.AddAllCurrentActions();
 }
示例#10
0
 public ActionManager(int noPrimitiveActions, MyModuleParams setup)
 {
     this.m_setup = setup;
     Actions      = new List <MyMotivatedAction>();
     for (int i = 0; i < noPrimitiveActions; i++)
     {
         Actions.Add(new MyMotivatedAction("" + i, 0, setup));
     }
     this.AcitonHistory = new MyActionHistory(this, setup);
     this.AcitonHistory.AddAllCurrentActions();
 }
 public ActionManager(int noPrimitiveActions, MyModuleParams setup)
 {
     this.m_setup = setup;
     Actions = new List<MyMotivatedAction>();
     for (int i = 0; i < noPrimitiveActions; i++)
     {
         Actions.Add(new MyMotivatedAction("" + i, 0, setup));
     }
     this.AcitonHistory = new MyActionHistory(this, setup);
     this.AcitonHistory.AddAllCurrentActions();
 }
示例#12
0
 public ActionManager(String[] labels, MyModuleParams setup)
 {
     this.m_setup = setup;
     Actions      = new List <MyMotivatedAction>();
     for (int i = 0; i < labels.Length; i++)
     {
         Actions.Add(new MyMotivatedAction(labels[i], 0, setup));
     }
     this.AcitonHistory = new MyActionHistory(this, setup);
     this.AcitonHistory.AddAllCurrentActions();
 }
        public MyAbstractHierarchy(MyRootDecisionSpace rds, MyModuleParams learningParams)
        {
            this.m_learningParams = learningParams;
            m_actionLevels        = new List <List <MyAction> >();

            this.m_rds = rds;
            for (int i = 0; i < rds.ActionManager.GetNoActions(); i++)
            {
                this.AddAction(i);
            }
        }
示例#14
0
        public MyLocalVariableHistory(MyRootDecisionSpace rds,
                                      MyModuleParams setup, IDecisionSpace ds)
        {
            m_rds   = rds;
            m_setup = setup;
            changes = new Dictionary <int, float>();

            // all potential variables added by default
            for (int i = 0; i < rds.VarManager.GetMaxVariables(); i++)
            {
                changes.Add(i, INIT_VAL);
            }
        }
示例#15
0
        public MyEligibilityTrace(MyModuleParams learning)
        {
            this.m_learning = learning;
            m_traceData     = new List <TraceData>();
            Lambda          = new float[learning.TraceLength];
            float tmp = learning.Lambda;

            for (int i = 0; i < learning.TraceLength; i++)
            {
                Lambda[i] = tmp;
                tmp      *= learning.Lambda;
            }
        }
        public MyStochasticReturnPredictor(MyRootDecisionSpace rds, int myPromotedVariable,
                                           MyModuleParams setup, String label, int level)
            : base(label, level, setup)
        {
            base.AddPromotedVariable(myPromotedVariable, rds);

            this.Rds             = rds;
            Ds                   = new MyDecisionSpace(this, rds, setup);
            Mem                  = new MyQSAMemory(rds.VarManager.MAX_VARIABLES, 0);
            m_asm                = new MyMotivationBasedDeleteUnselectedASM(setup);
            LearningAlgorithm    = new MyDiscreteQLearning(setup, Mem);
            m_mlvh               = new MyLocalVariableHistory(rds, m_setup, Ds);
            m_prevSelectedAction = 0;
            m_prev_st            = Ds.GetCurrentState();
            this.m_newVariables  = new List <int>();
        }
        private int[] m_prev_st; // previous state (for variable adding and sharing knowledge)

        #endregion Fields

        #region Constructors

        public MyStochasticReturnPredictor(MyRootDecisionSpace rds, int myPromotedVariable,
            MyModuleParams setup, String label, int level)
            : base(label, level, setup)
        {
            base.AddPromotedVariable(myPromotedVariable, rds);

            this.Rds = rds;
            Ds = new MyDecisionSpace(this, rds, setup);
            Mem = new MyQSAMemory(rds.VarManager.MAX_VARIABLES, 0);
            m_asm = new MyMotivationBasedDeleteUnselectedASM(setup);
            LearningAlgorithm = new MyDiscreteQLearning(setup, Mem);
            m_mlvh = new MyLocalVariableHistory(rds, m_setup, Ds);
            m_prevSelectedAction = 0;
            m_prev_st = Ds.GetCurrentState();
            this.m_newVariables = new List<int>();
        }
示例#18
0
        public VariableManager(String[] labels, MyModuleParams setup)
        {
            this.m_labels = labels;
            m_vars        = new MyVariable[labels.Length];

            if (MAX_VARIABLES <= 0)
            {
                MAX_VARIABLES = 1;
            }
            else
            {
                this.MAX_VARIABLES = labels.Length;
            }
            this.InitVars();
            this.VarHistory = new MyVariableHistory(this, setup);
            this.VarHistory.AddAllPotentialVariables();
        }
示例#19
0
        public VariableManager(int maxVariables, MyModuleParams setup)
        {
            if (maxVariables <= 0)
            {
                maxVariables = 1;
            }
            else
            {
                this.MAX_VARIABLES = maxVariables;
            }
            m_labels = new String[this.MAX_VARIABLES];

            for (int i = 0; i < m_labels.Length; i++)
            {
                m_labels[i] = "" + i;
            }
            this.InitVars();
            this.VarHistory = new MyVariableHistory(this, setup);
            this.VarHistory.AddAllPotentialVariables();

            this.ShouldBeSubspaced = new List <int>();
        }
 public MyMotivationBasedDeleteUnselectedASM(MyModuleParams setup)
 {
     this.m_setup = setup;
     m_rnd = new Random();
 }
 public MyMotivatedAction(String label, int level, MyModuleParams setup)
     : base(label, level)
 {
     this.m_motivatonSource = new MyMotivationSource(setup, level);
     this.m_setup = setup;
 }
示例#22
0
 public MyRootDecisionSpace(int maxVariables, String[] actionNames, MyModuleParams setup)
 {
     VarManager    = new VariableManager(maxVariables, setup);
     ActionManager = new ActionManager(actionNames, setup);
 }
示例#23
0
 public MyMotivatedAction(String label, int level, MyModuleParams setup)
     : base(label, level)
 {
     this.m_motivatonSource = new MyMotivationSource(setup, level);
     this.m_setup           = setup;
 }
示例#24
0
 public MyMotivationSource(MyModuleParams setup, int level)
 {
     this.m_setup = setup;
     this.m_val = 0;
     this.m_level = level;
 }
 public MyRootDecisionSpace(int maxVariables, String[] actionNames, MyModuleParams setup)
 {
     VarManager = new VariableManager(maxVariables, setup);
     ActionManager = new ActionManager(actionNames, setup);
 }
        public MyAbstractHierarchy(MyRootDecisionSpace rds, MyModuleParams learningParams)
        {
            this.m_learningParams = learningParams;
            m_actionLevels = new List<List<MyAction>>();

            this.m_rds = rds;
            for (int i = 0; i < rds.ActionManager.GetNoActions(); i++)
            {
                this.AddAction(i);
            }
        }
 public MyHierarchyMaintainer(MyRootDecisionSpace rds, MyModuleParams learningParams)
     : base(rds, learningParams)
 {
 }
示例#28
0
 public MyMotivationBasedDeleteUnselectedASM(MyModuleParams setup)
 {
     this.m_setup = setup;
     m_rnd        = new Random();
 }
        public VariableManager(int maxVariables, MyModuleParams setup)
        {
            if (maxVariables <= 0)
            {
                maxVariables = 1;
            }
            else
            {
                this.MAX_VARIABLES = maxVariables;
            }
            m_labels = new String[this.MAX_VARIABLES];

            for (int i = 0; i < m_labels.Length; i++)
            {
                m_labels[i] = "" + i;
            }
            this.InitVars();
            this.VarHistory = new MyVariableHistory(this, setup);
            this.VarHistory.AddAllPotentialVariables();

            this.ShouldBeSubspaced = new List<int>();
        }
        public VariableManager(String[] labels, MyModuleParams setup)
        {
            this.m_labels = labels;
            m_vars = new MyVariable[labels.Length];

            if (MAX_VARIABLES <= 0)
            {
                MAX_VARIABLES = 1;
            }
            else
            {
                this.MAX_VARIABLES = labels.Length;
            }
            this.InitVars();
            this.VarHistory = new MyVariableHistory(this, setup);
            this.VarHistory.AddAllPotentialVariables();
        }
        private List<TraceData> m_traceData; // position 0 stores the most recent state-aciton

        #endregion Fields

        #region Constructors

        public MyEligibilityTrace(MyModuleParams learning)
        {
            this.m_learning = learning;
            m_traceData = new List<TraceData>();
            Lambda = new float[learning.TraceLength];
            float tmp = learning.Lambda;

            for (int i = 0; i < learning.TraceLength; i++)
            {
                Lambda[i] = tmp;
                tmp *= learning.Lambda;
            }
        }
        public MyLocalVariableHistory(MyRootDecisionSpace rds,
            MyModuleParams setup, IDecisionSpace ds)
        {
            m_rds = rds;
            m_setup = setup;
            changes = new Dictionary<int, float>();

            // all potential variables added by default
            for (int i = 0; i < rds.VarManager.GetMaxVariables(); i++)
            {
                changes.Add(i, INIT_VAL);
            }
        }
        public MyVariableHistory(VariableManager vm, MyModuleParams setup)
        {
            this.vm = vm;
            this.setup = setup;

            this.changed = new List<List<int>>();

            isMonitored = new bool[vm.GetMaxVariables()];
        }
示例#34
0
 public MyMotivationSource(MyModuleParams setup, int level)
 {
     this.m_setup = setup;
     this.m_val   = 0;
     this.m_level = level;
 }
 public MyAbsHierarchyMaintainer(MyRootDecisionSpace rds, MyModuleParams learningParams)
     : base(rds, learningParams)
 {
 }