private void UpdateRequirementMaps(Sequence <CompoundTerm> actions, IState iState)
{
Set <string> yesStrings = new Set <string>();
foreach (Action a in actions)
{
System.Console.WriteLine(iState.GetHashCode() + a.ToString());
foreach (string s in this.modelProgram.GetEnablingConditionDescriptions(iState, a, false))
{
yesStrings = yesStrings.Add(s);
if (!requirementProperties.Contains(s))
{
requirementProperties = requirementProperties.AddLast(s);
requireEnabledStateMap = requireEnabledStateMap.Add(s, Set <int> .EmptySet);
}
requireEnabledStateMap = requireEnabledStateMap.Override(s, requireEnabledStateMap[s].Add(iState.GetHashCode()));
}
}
bdt = bdt.addState(yesStrings, iState.GetHashCode());
if (i == 0)
{
if (i < requirementProperties.Count)
{
string s1 = requirementProperties[i++];
bdt = bdt.Refine(s1, requireEnabledStateMap[s1]);
bdt.PrintTree(0);
}
}
}
private void doValueIteration(BinaryDecisionTree bdt)
{
List <double> maxV = bdt.ReturnValue(true);
List <double> minV = bdt.ReturnValue(false);
List <Set <int> > abstractMap = bdt.ReturnLeaves();
double tmp;
int maxvcount = maxV.Count;
double diff = 1.0;
double epsilon = 0.1;
while (diff > epsilon)
{
diff = 0.0;
for (int k = 0; k < maxvcount; k++)
{
tmp = doLocalValueIteration(k, abstractMap, maxV, true);
diff = Math.Max(diff, Math.Abs(tmp - maxV[k]));
maxV[k] = tmp;
tmp = doLocalValueIteration(k, abstractMap, minV, false);
diff = Math.Max(diff, Math.Abs(tmp - minV[k]));
minV[k] = tmp;
}
}
for (int k = 0; k < maxvcount; k++)
{
System.Console.WriteLine("maxv " + maxV[k] + " minv " + minV[k]);
}
// Need to update the maxValue as well as minValue in the tree
bdt = bdt.UpdateMaxMin(maxV, minV, abstractMap);
}
public MDPNewAbstractStrategy(ModelProgram modelProgram) :
base(modelProgram)
{
requirementProperties = Sequence <string> .EmptySequence;
requireEnabledStateMap = Map <string, Set <int> > .EmptyMap;
activeEdges = new Dictionary <int, Set <int> >();
passiveEdges = new Dictionary <int, Bag <int> >();
bdt = new BinaryDecisionTree("root", null, null, 1, 1, new Set <int>(currState.GetHashCode()));
}
public BinaryDecisionTree(string prop, BinaryDecisionTree t, BinaryDecisionTree f, double maxv, double minv, Set <int> setStates)
{
this.property = prop;
this.trueEdge = t;
this.falseEdge = f;
this.maxValue = maxv;
this.minValue = minv;
this.states = setStates;
}
public BinaryDecisionTree(string prop, BinaryDecisionTree t, BinaryDecisionTree f, double maxv, double minv, Set<int> setStates)
{
this.property = prop;
this.trueEdge = t;
this.falseEdge = f;
this.maxValue = maxv;
this.minValue = minv;
this.states = setStates;
}
internal BinaryDecisionTree UpdateMaxMin(List <double> maxV, List <double> minV, List <Set <int> > abstractMap)
{
if (this.trueEdge == null && this.falseEdge == null)
{
if (abstractMap.Contains(states))
{
int index = abstractMap.IndexOf(states);
this.maxValue = maxV[index];
this.minValue = minV[index];
}
return(this);
}
else
{
trueEdge = trueEdge.UpdateMaxMin(maxV, minV, abstractMap);
falseEdge = falseEdge.UpdateMaxMin(maxV, minV, abstractMap);
return(this);
}
}
public BinaryDecisionTree addState(Set <string> trueProps, int StateId)
{
if (states.Contains(StateId))
{
return(this);
}
//if (this.trueEdge == null && this.falseEdge == null)
//{
states = this.states.Add(StateId);
//}
if (trueEdge != null && trueProps.Contains(this.property))
{
this.trueEdge = this.trueEdge.addState(trueProps, StateId);
}
else if (falseEdge != null)
{
this.falseEdge = this.falseEdge.addState(trueProps, StateId);
}
return(this);
}
public BinaryDecisionTree Refine(string prop, Set <int> enabledStates)
{
// dont refine already added property
if (this.property.Equals(prop))
{
return(this);
}
if (states.IsEmpty)
{
return(this);
}
Set <int> s1 = states.Intersect(enabledStates);
Set <int> s2 = states.Difference(enabledStates);
if (this.trueEdge != null)
{
this.trueEdge.Refine(prop, s1);
}
if (this.falseEdge != null)
{
this.falseEdge.Refine(prop, s2);
}
if (this.trueEdge == null && this.falseEdge == null)
{
//if (!s1.IsEmpty)
//{
this.trueEdge = new BinaryDecisionTree(prop, null, null, this.maxValue, this.minValue, s1);
//}
//else this.trueEdge = null;
//if (!s2.IsEmpty)
//{
this.falseEdge = new BinaryDecisionTree(prop, null, null, this.maxValue, this.minValue, s2);
//}
//else this.falseEdge = null;
}
return(this);
}
private CompoundTerm ChooseAction(Sequence <Action> actions, IState iState)
{
TransitionProperties tp;
int targetId = -1;
List <double> MaxV = bdt.ReturnValue(true);
List <double> MinV = bdt.ReturnValue(false);
List <Set <int> > abstractMap = bdt.ReturnLeaves();
Sequence <Pair <int, Action> > cumulActSum = Sequence <Pair <int, Action> > .EmptySequence;
double epsilon = 0.1;
Dictionary <int, int> sumTarget = new Dictionary <int, int>();
Action maxAct = null;
int targetAbsId = -1;
int sum = 0;
UpdateRequirementMaps(actions, iState);
Set <Action> newStateActs = new Set <Action>();
Set <Action> oldActs = new Set <Action>(actions.Head);
foreach (Action a in actions)
{
int tState = this.modelProgram.GetTargetState(iState, a, null, out tp).GetHashCode();
targetAbsId = findAbstractId(tState, abstractMap);
if (targetAbsId == -1)
{
newStateActs = newStateActs.Add(a);
}
else
{
sum = sum + (int)(MaxV[targetAbsId] * Math.Pow(10.0, 9.0));
Pair <int, Action> np = new Pair <int, Action>(sum, a);
sumTarget.Add(sum, targetAbsId);
cumulActSum = cumulActSum.AddLast(np);
}
}
if (!newStateActs.IsEmpty)
{
maxAct = newStateActs.Choose();
System.Console.WriteLine("new action in new state " + maxAct.ToString());
return(maxAct);
}
else
{
Random rndNumbers = new Random();
int rndNumber = rndNumbers.Next(sum);
System.Console.WriteLine(sum + " " + rndNumber);
foreach (Pair <int, Action> np in cumulActSum)
{
System.Console.WriteLine(np.First + " " + np.Second.ToString());
if (rndNumber <= np.First)
{
maxAct = np.Second;
targetId = sumTarget[np.First];
break;
}
targetId = sumTarget[np.First];
maxAct = np.Second;
}
System.Console.WriteLine("old action in old state " + maxAct.ToString());
}
// Adaptive Refinement
if (MaxV[targetId] - MinV[targetId] > epsilon)
{
if (i < requirementProperties.Count)
{
string s1 = requirementProperties[i++];
bdt = bdt.Refine(s1, requireEnabledStateMap[s1]);
bdt.PrintTree(0);
}
}
return(maxAct);
}
public BinaryDecisionTree Refine(string prop, Set<int> enabledStates)
{
// dont refine already added property
if (this.property.Equals(prop))
return this;
if (states.IsEmpty)
return this;
Set<int> s1 = states.Intersect(enabledStates);
Set<int> s2 = states.Difference(enabledStates);
if (this.trueEdge != null) this.trueEdge.Refine(prop,s1);
if (this.falseEdge != null) this.falseEdge.Refine(prop,s2);
if (this.trueEdge == null && this.falseEdge == null)
{
//if (!s1.IsEmpty)
//{
this.trueEdge = new BinaryDecisionTree(prop, null, null,this.maxValue,this.minValue, s1);
//}
//else this.trueEdge = null;
//if (!s2.IsEmpty)
//{
this.falseEdge = new BinaryDecisionTree(prop, null, null,this.maxValue,this.minValue, s2);
//}
//else this.falseEdge = null;
}
return this;
}
private void UpdateRequirementMaps(Sequence<CompoundTerm> actions, IState iState)
{
Set<string> yesStrings = new Set<string>();
foreach (Action a in actions)
{
System.Console.WriteLine(iState.GetHashCode() + a.ToString());
foreach (string s in this.modelProgram.GetEnablingConditionDescriptions(iState, a, false))
{
yesStrings = yesStrings.Add(s);
if (!requirementProperties.Contains(s))
{
requirementProperties = requirementProperties.AddLast(s);
requireEnabledStateMap = requireEnabledStateMap.Add(s, Set<int>.EmptySet);
}
requireEnabledStateMap = requireEnabledStateMap.Override(s, requireEnabledStateMap[s].Add(iState.GetHashCode()));
}
}
bdt = bdt.addState(yesStrings, iState.GetHashCode());
if (i == 0)
{
if (i < requirementProperties.Count)
{
string s1 = requirementProperties[i++];
bdt = bdt.Refine(s1, requireEnabledStateMap[s1]);
bdt.PrintTree(0);
}
}
}
private CompoundTerm ChooseAction(Sequence<Action> actions, IState iState)
{
TransitionProperties tp;
int targetId = -1;
List<double> MaxV = bdt.ReturnValue(true);
List<double> MinV = bdt.ReturnValue(false);
List<Set<int>> abstractMap = bdt.ReturnLeaves();
Sequence<Pair<int, Action>> cumulActSum = Sequence<Pair<int, Action>>.EmptySequence;
double epsilon = 0.1;
Dictionary<int, int> sumTarget = new Dictionary<int, int>();
Action maxAct = null;
int targetAbsId = -1;
int sum = 0;
UpdateRequirementMaps(actions, iState);
Set<Action> newStateActs = new Set<Action>();
Set<Action> oldActs = new Set<Action>(actions.Head);
foreach (Action a in actions)
{
int tState = this.modelProgram.GetTargetState(iState, a, null, out tp).GetHashCode();
targetAbsId = findAbstractId(tState, abstractMap);
if (targetAbsId == -1)
{
newStateActs = newStateActs.Add(a);
}
else
{
sum = sum + (int)(MaxV[targetAbsId] * Math.Pow(10.0, 9.0));
Pair<int, Action> np = new Pair<int, Action>(sum, a);
sumTarget.Add(sum, targetAbsId);
cumulActSum = cumulActSum.AddLast(np);
}
}
if (!newStateActs.IsEmpty)
{
maxAct = newStateActs.Choose();
System.Console.WriteLine("new action in new state " + maxAct.ToString());
return maxAct;
}
else
{
Random rndNumbers = new Random();
int rndNumber = rndNumbers.Next(sum);
System.Console.WriteLine(sum + " " + rndNumber);
foreach (Pair<int, Action> np in cumulActSum)
{
System.Console.WriteLine(np.First + " " + np.Second.ToString());
if (rndNumber <= np.First)
{
maxAct = np.Second;
targetId = sumTarget[np.First];
break;
}
targetId = sumTarget[np.First];
maxAct = np.Second;
}
System.Console.WriteLine("old action in old state " + maxAct.ToString());
}
// Adaptive Refinement
if (MaxV[targetId] - MinV[targetId] > epsilon)
{
if (i < requirementProperties.Count)
{
string s1 = requirementProperties[i++];
bdt = bdt.Refine(s1, requireEnabledStateMap[s1]);
bdt.PrintTree(0);
}
}
return maxAct;
}
private void doValueIteration(BinaryDecisionTree bdt)
{
List<double> maxV = bdt.ReturnValue(true);
List<double> minV = bdt.ReturnValue(false);
List<Set<int>> abstractMap = bdt.ReturnLeaves();
double tmp;
int maxvcount = maxV.Count;
double diff = 1.0;
double epsilon = 0.1;
while (diff > epsilon)
{
diff = 0.0;
for (int k = 0; k < maxvcount;k++)
{
tmp = doLocalValueIteration(k, abstractMap, maxV, true);
diff = Math.Max(diff, Math.Abs(tmp - maxV[k]));
maxV[k] = tmp;
tmp = doLocalValueIteration(k, abstractMap, minV, false);
diff = Math.Max(diff, Math.Abs(tmp - minV[k]));
minV[k] = tmp;
}
}
for (int k = 0; k < maxvcount; k++)
{
System.Console.WriteLine("maxv "+ maxV[k]+ " minv "+ minV[k]);
}
// Need to update the maxValue as well as minValue in the tree
bdt = bdt.UpdateMaxMin(maxV, minV, abstractMap);
}
public MDPNewAbstractStrategy(ModelProgram modelProgram):
base(modelProgram)
{
requirementProperties = Sequence<string>.EmptySequence;
requireEnabledStateMap = Map<string, Set<int>>.EmptyMap;
activeEdges = new Dictionary<int, Set<int>>();
passiveEdges = new Dictionary<int, Bag<int>>();
bdt = new BinaryDecisionTree("root", null, null,1,1, new Set<int>(currState.GetHashCode()));
}
internal BinaryDecisionTree UpdateMaxMin(List<double> maxV, List<double> minV, List<Set<int>> abstractMap)
{
if (this.trueEdge == null && this.falseEdge == null)
{
if (abstractMap.Contains(states))
{
int index = abstractMap.IndexOf(states);
this.maxValue = maxV[index];
this.minValue = minV[index];
}
return this;
}
else
{
trueEdge = trueEdge.UpdateMaxMin(maxV, minV, abstractMap);
falseEdge = falseEdge.UpdateMaxMin(maxV, minV, abstractMap);
return this;
}
}
public BinaryDecisionTree addState(Set<string> trueProps, int StateId)
{
if (states.Contains(StateId))
return this;
//if (this.trueEdge == null && this.falseEdge == null)
//{
states = this.states.Add(StateId);
//}
if (trueEdge!=null && trueProps.Contains(this.property))
{
this.trueEdge = this.trueEdge.addState(trueProps, StateId);
}
else if(falseEdge != null)
this.falseEdge = this.falseEdge.addState(trueProps, StateId);
return this;
}