public override void UpdateMemoryBlocks()
{
UtilityOutput.Count = NoActions;
LearningParams = new MyModuleParams();
if (GlobalDataInput != null)
{
if (NoActions == 6)
{
MyLog.DEBUG.WriteLine("6 actions set by the user, will use action names for gridworld");
Rds = new MyRootDecisionSpace(GlobalDataInput.Count, new String[] { " -", " <", " >", " ^", " v", " P" }, LearningParams);
}
else if (NoActions == 3)
{
MyLog.DEBUG.WriteLine("3 actions set by the user, will use action names for pong");
Rds = new MyRootDecisionSpace(GlobalDataInput.Count, new String[] { " <", " -", " >" }, LearningParams);
}
else
{
MyLog.DEBUG.WriteLine("Unknown no. of actions, will use automatic naming of actions");
String[] names = new String[NoActions];
for (int i = 0; i < NoActions; i++)
{
names[i] = "A" + i;
}
Rds = new MyRootDecisionSpace(GlobalDataInput.Count, names, LearningParams);
}
Hierarchy = new MyHierarchyMaintainer(Rds, LearningParams);
SubspaceUtils.Count = MaxActions * MaxActions;
SubspaceUtils.ColumnHint = MaxActions;
}
}
public override void UpdateMemoryBlocks()
{
UtilityOutput.Count = NoActions;
RewardStats.Count = 2;
LearningParams = new MyModuleParams();
Memory = new MyQSAMemory(GlobalDataInput.Count, NoActions);
LearningAlgorithm = new MyDiscreteQLearning(LearningParams, Memory);
if (GlobalDataInput != null)
{
if (NoActions == 6)
{
MyLog.DEBUG.WriteLine("6 actions set by the user, will use action names for gridworld");
Rds = new MyRootDecisionSpace(GlobalDataInput.Count, new String[] { " -", " <", " >", " ^", " v", " P" }, LearningParams);
}
else if (NoActions == 3)
{
MyLog.DEBUG.WriteLine("3 actions set by the user, will use action names for pong");
Rds = new MyRootDecisionSpace(GlobalDataInput.Count, new String[] { " <", " -", " >" }, LearningParams);
}
else
{
MyLog.DEBUG.WriteLine("Unknown no. of actions, will use automatic naming of actions");
String[] names = new String[NoActions];
for (int i = 0; i < NoActions; i++)
{
names[i] = "A" + i;
}
Rds = new MyRootDecisionSpace(GlobalDataInput.Count, names, LearningParams);
}
CurrentStateOutput.Count = GlobalDataInput.Count;
}
}
public MyDecisionSpace(MyStochasticReturnPredictor mySRP, MyRootDecisionSpace rds, MyModuleParams setup)
{
this.m_setup = setup;
this.m_rds = rds;
this.m_mySRP = mySRP;
m_S_t = new int[rds.VarManager.MAX_VARIABLES];
ChildActions = new MyActionLink(0, rds);
m_childVariables = new MyVariableLink(rds.VarManager.GetMaxVariables());
}
public MyAbstractHierarchy(MyRootDecisionSpace rds, MyModuleParams learningParams)
{
this.m_learningParams = learningParams;
m_actionLevels = new List <List <MyAction> >();
this.m_rds = rds;
for (int i = 0; i < rds.ActionManager.GetNoActions(); i++)
{
this.AddAction(i);
}
}
public MyLocalVariableHistory(MyRootDecisionSpace rds,
MyModuleParams setup, IDecisionSpace ds)
{
m_rds = rds;
m_setup = setup;
changes = new Dictionary <int, float>();
// all potential variables added by default
for (int i = 0; i < rds.VarManager.GetMaxVariables(); i++)
{
changes.Add(i, INIT_VAL);
}
}
public MyStochasticReturnPredictor(MyRootDecisionSpace rds, int myPromotedVariable,
MyModuleParams setup, String label, int level)
: base(label, level, setup)
{
base.AddPromotedVariable(myPromotedVariable, rds);
this.Rds = rds;
Ds = new MyDecisionSpace(this, rds, setup);
Mem = new MyQSAMemory(rds.VarManager.MAX_VARIABLES, 0);
m_asm = new MyMotivationBasedDeleteUnselectedASM(setup);
LearningAlgorithm = new MyDiscreteQLearning(setup, Mem);
m_mlvh = new MyLocalVariableHistory(rds, m_setup, Ds);
m_prevSelectedAction = 0;
m_prev_st = Ds.GetCurrentState();
this.m_newVariables = new List <int>();
}
private int[] m_prev_st; // previous state (for variable adding and sharing knowledge)
#endregion Fields
#region Constructors
public MyStochasticReturnPredictor(MyRootDecisionSpace rds, int myPromotedVariable,
MyModuleParams setup, String label, int level)
: base(label, level, setup)
{
base.AddPromotedVariable(myPromotedVariable, rds);
this.Rds = rds;
Ds = new MyDecisionSpace(this, rds, setup);
Mem = new MyQSAMemory(rds.VarManager.MAX_VARIABLES, 0);
m_asm = new MyMotivationBasedDeleteUnselectedASM(setup);
LearningAlgorithm = new MyDiscreteQLearning(setup, Mem);
m_mlvh = new MyLocalVariableHistory(rds, m_setup, Ds);
m_prevSelectedAction = 0;
m_prev_st = Ds.GetCurrentState();
this.m_newVariables = new List<int>();
}
public MyHierarchyMaintainer(MyRootDecisionSpace rds, MyModuleParams learningParams)
: base(rds, learningParams)
{
}
/// <summary>
/// Works with only one variable
/// </summary>
public void AddPromotedVariable(int varIndex, MyRootDecisionSpace rds)
{
m_myPromotedVariableInd = varIndex;
rds.VarManager.GetVarNo(varIndex).SetMyAction((MyStochasticReturnPredictor)this);
}
public MyLocalVariableHistory(MyRootDecisionSpace rds,
MyModuleParams setup, IDecisionSpace ds)
{
m_rds = rds;
m_setup = setup;
changes = new Dictionary<int, float>();
// all potential variables added by default
for (int i = 0; i < rds.VarManager.GetMaxVariables(); i++)
{
changes.Add(i, INIT_VAL);
}
}
/// <summary>
/// For a given predictor, the method creates 2D array of max action utilities and max action labels over selected dimensions.
/// The values in the memory are automatically scaled into the interval 0,1. Realtime values are multililed by motivations (therfore are bigger).
/// </summary>
/// <param name="values">array passed by reference for storing utilities of best action</param>
/// <param name="labelIndexes">array of the same size for best action indexes</param>
/// <param name="predictor">an asbtract action</param>
/// <param name="XVarIndex">global index of state variable in the VariableManager</param>
/// <param name="YVarIndex">the same: y axis</param>
/// <param name="showRealtimeUtilities">show current utilities (scaled by motivations from the source and the hierarchy?)</param>
public void ReadTwoDimensions(ref float[,] values, ref int[,] labelIndexes,
MyStochasticReturnPredictor predictor, int XVarIndex, int YVarIndex, bool showRealtimeUtilities)
{
MyRootDecisionSpace rds = predictor.Rds;
if (XVarIndex >= rds.VarManager.MAX_VARIABLES)
{
XVarIndex = rds.VarManager.MAX_VARIABLES - 1;
}
if (YVarIndex >= rds.VarManager.MAX_VARIABLES)
{
YVarIndex = rds.VarManager.MAX_VARIABLES - 1;
}
if (YVarIndex < 0)
{
YVarIndex = 0;
}
if (XVarIndex < 0)
{
XVarIndex = 0;
}
MyQSAMemory mem = predictor.Mem;
int[] sizes = mem.GetStateSizes(); // size of the matrix
int[] indexes = predictor.Ds.GetCurrentState(); // initial indexes
int[] actionGlobalIndexes = mem.GetGlobalActionIndexes(); // global indexes of actions in the memory
int promotedIndex = predictor.GetPromotedVariableIndex();
MyVariable varX = rds.VarManager.GetVarNo(XVarIndex);
MyVariable varY = rds.VarManager.GetVarNo(YVarIndex);
float[] varXvals = varX.Values.ToArray();
float[] varYvals = varY.Values.ToArray();
Array.Sort(varXvals);
Array.Sort(varYvals);
int sx = 0;
int sy = 0;
if (XVarIndex == promotedIndex)
{
sx = 1;
indexes[XVarIndex] = 0;
varXvals = new float[] { 0 };
sy = this.ReadSize(predictor.Ds, varY, YVarIndex, predictor.GetLabel());
}
else if (YVarIndex == promotedIndex)
{
sy = 1;
indexes[YVarIndex] = 0;
varYvals = new float[] { 0 };
sx = this.ReadSize(predictor.Ds, varX, XVarIndex, predictor.GetLabel());
}
else
{
sx = this.ReadSize(predictor.Ds, varX, XVarIndex, predictor.GetLabel());
sy = this.ReadSize(predictor.Ds, varY, YVarIndex, predictor.GetLabel());
}
if (values == null || labelIndexes == null ||
values.GetLength(0) != sx || values.GetLength(1) != sy ||
labelIndexes.GetLength(0) != sx || labelIndexes.GetLength(1) != sy)
{
values = new float[sx, sy];
labelIndexes = new int[sx, sy];
}
for (int i = 0; i < sx; i++)
{
indexes[XVarIndex] = (int)varXvals[i];
for (int j = 0; j < sy; j++)
{
indexes[YVarIndex] = (int)varYvals[j];
float[] utilities = mem.ReadData(indexes);
if (predictor.GetMaxMemoryValue() != 0)
{
for (int k = 0; k < utilities.Length; k++)
{
utilities[k] = utilities[k] / predictor.GetMaxMemoryValue();
}
}
float maxValue = 0.0f;
int maxIndex = 0;
if (utilities.Length != actionGlobalIndexes.Length)
{
MyLog.DEBUG.WriteLine("ERROR: unexpected length of utilities array, will place default values");
utilities = new float[actionGlobalIndexes.Length];
}
else if (actionGlobalIndexes.Length == 0)
{
MyLog.DEBUG.WriteLine("WARNING: this DS contains no actions. Will use the action 0");
utilities = new float[1];
actionGlobalIndexes = new int[] { 0 };
}
else
{
maxValue = utilities.Max();
maxIndex = utilities.ToList().IndexOf(maxValue);
}
if (showRealtimeUtilities)
{
values[i, j] = maxValue * predictor.GetMyTotalMotivation();
}
else
{
values[i, j] = maxValue;
}
labelIndexes[i, j] = actionGlobalIndexes[maxIndex];
}
}
}
public MyAbsHierarchyMaintainer(MyRootDecisionSpace rds, MyModuleParams learningParams)
: base(rds, learningParams)
{
}
public MyAbstractHierarchy(MyRootDecisionSpace rds, MyModuleParams learningParams)
{
this.m_learningParams = learningParams;
m_actionLevels = new List<List<MyAction>>();
this.m_rds = rds;
for (int i = 0; i < rds.ActionManager.GetNoActions(); i++)
{
this.AddAction(i);
}
}
public MyActionLink(int initSize, MyRootDecisionSpace rds)
{
this.m_rds = rds;
m_childIncluded = new List <bool>(initSize);
m_weights = new List <float>(initSize);
}
/// <summary>
/// Works with only one variable
/// </summary>
/// <param name="varIndex"></param>
public void AddPromotedVariable(int varIndex, MyRootDecisionSpace rds)
{
m_myPromotedVariableInd = varIndex;
rds.VarManager.GetVarNo(varIndex).SetMyAction((MyStochasticReturnPredictor)this);
}