public Distribution DistrbutionReCalculate(HashSet<string> scc, MDPState node, Distribution distr)
{
double nonSelfLoopProb = 0.0;
for (int i = 0; i < distr.States.Count; i++)
{
KeyValuePair<double, MDPState> pair = distr.States[i];
if (scc.Contains(pair.Value.ID))
{
//selfLoopProb += pair.Key;
//hasSelfLoop = true;
//the self loop is removed in this distribution
distr.States.Remove(pair);
//i-- is used to keep the loop correct after removing one element
i--;
}
else
{
nonSelfLoopProb += pair.Key;
}
}
foreach (KeyValuePair<double, MDPState> pair in distr.States)
{
//KeyValuePair<double, MDPState> newPair = new KeyValuePair<double, MDPState>(pair.Key / nonSelfLoopProb, pair.Value);
distr.AddProbStatePair(pair.Key / nonSelfLoopProb, pair.Value);
distr.States.Remove(pair);
}
return distr;
}
public void Test_QLearning_Path_Finder()
{
// start
var master = new MDPState(2);
var kitchen = new MDPState(3);
master.Successors.Add(new MDPSuccessorState(new AI.Action(1, "Goto Kitchen"), 0.1, kitchen, 0));
var entrance = new MDPState(1);
var lounge = new MDPState(4);
kitchen.Successors.Add(new MDPSuccessorState(new AI.Action(2, "Goto Lounge"), 0.1, lounge, -15));
kitchen.Successors.Add(new MDPSuccessorState(new AI.Action(3, "Goto Entrance Hall"), 0, entrance, -30));
var spare = new MDPState(0);
lounge.Successors.Add(new MDPSuccessorState(new AI.Action(4, "Goto Spare Room"), 0.1, spare, -10));
var outside = new MDPState(5);
lounge.Successors.Add(new MDPSuccessorState(new AI.Action(5, "Go Outside"), 0.1, outside, 30));
entrance.Successors.Add(new MDPSuccessorState(new AI.Action(6, "Go Outside"), 0.1, outside, 50));
outside.Successors.Add(new MDPSuccessorState(new AI.Action(7, "Stay Outside"), 0.2, outside, 50));
var examples = MDPConverter.ToExamples(master);
Assert.Equal(7, examples.Item1.Rows);
Assert.Equal(7, examples.Item2.Length);
Assert.Equal(7, examples.Item3.Rows);
Assert.Equal(7, examples.Item4.Length);
var generator = new Reinforcement.QLearning.QLearnerGenerator() { Lambda = 0.9 };
Reinforcement.QLearning.QLearnerModel model = (Reinforcement.QLearning.QLearnerModel) generator.Generate(examples.Item1, examples.Item2, examples.Item3, examples.Item4);
Assert.Equal(3, (int) model.Predict(kitchen.ToVector())/*, "Expected to move from kitchen to entrance hall"*/);
Assert.Equal(5, (int) model.Predict(lounge.ToVector())/*, "Expected to move from lounge to outside"*/);
Assert.Equal(7, (int) model.Predict(outside.ToVector())/*, "Expected to stay outside"*/);
string path = "Start: " + master.Id; IMDPState current = master;
int counter = 0;
while (current.Id != outside.Id)
{
if (counter > 20) break;
double v = model.Predict(current.ToVector());
var next = current.GetSuccessors().Where(w => w.Action.Id == (int) v).FirstOrDefault() as IMDPSuccessor;
if (next == null) break;
current = next.State as IMDPState;
counter++;
path += $"\n next: { current.Id } ({ next.Reward.ToString("N2") })";
}
Console.Write(path);
}
private void BuildMDPSafety()
{
Stack<KeyValuePair<EventBAPairSafetyPCSP, MDPState>> working = new Stack<KeyValuePair<EventBAPairSafetyPCSP, MDPState>>(1024);
EventBAPairSafetyPCSP initialstep = EventBAPairSafetyPCSP.GetInitialPairs(BA, InitialStep as MDPConfiguration);
string initID = initialstep.GetID();
MDPState init = new MDPState(initID);
working.Push(new KeyValuePair<EventBAPairSafetyPCSP, MDPState>(initialstep, init));
mdp = new MDP(init, Ultility.Ultility.DEFAULT_PRECISION, Ultility.Ultility.MAX_DIFFERENCE);
do
{
if (CancelRequested)
{
VerificationOutput.NoOfStates = mdp.States.Count;
}
KeyValuePair<EventBAPairSafetyPCSP, MDPState> current = working.Pop();
if (current.Key.BAStates.Count == 0)
{
mdp.AddTargetStates(current.Value);
}
else
{
MDPConfiguration[] steps = current.Key.Config.MakeOneMoveLocal().ToArray();
this.VerificationOutput.Transitions += steps.Length;
List<EventBAPairSafetyPCSP> products = EventBAPairSafetyPCSP.Next(BA, steps, current.Key.BAStates);
int currentDistriIndex = -1;
Distribution newDis = new Distribution(Constants.TAU);
foreach (EventBAPairSafetyPCSP eventBaPairSafetyPcsp in products)
{
string stepID = eventBaPairSafetyPcsp.GetID();
MDPState nextState;
if (!mdp.States.TryGetValue(stepID, out nextState))
{
nextState = new MDPState(stepID);
mdp.AddState(nextState);
working.Push(new KeyValuePair<EventBAPairSafetyPCSP, MDPState>(eventBaPairSafetyPcsp, nextState));
}
if (eventBaPairSafetyPcsp.Config.DisIndex == -1)
{
if (currentDistriIndex >= 0)
{
current.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
}
Distribution newTrivialDis = new Distribution(eventBaPairSafetyPcsp.Config.Event);
newTrivialDis.AddProbStatePair(1, nextState);
current.Value.AddDistribution(newTrivialDis);
}
else if (currentDistriIndex != -1 && eventBaPairSafetyPcsp.Config.DisIndex != currentDistriIndex)
{
current.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
newDis.AddProbStatePair(eventBaPairSafetyPcsp.Config.Probability, nextState);
}
else
{
newDis.AddProbStatePair(eventBaPairSafetyPcsp.Config.Probability, nextState);
}
currentDistriIndex = eventBaPairSafetyPcsp.Config.DisIndex;
}
if (currentDistriIndex >= 0)
{
current.Value.AddDistribution(newDis);
}
}
} while (working.Count > 0);
VerificationOutput.NoOfStates = mdp.States.Count;
//mdp.BackUpTargetStates();
}
protected MDP BuildMDP()
{
Stack<KeyValuePair<MDPConfiguration, MDPState>> working = new Stack<KeyValuePair<MDPConfiguration, MDPState>>(1024);
string initID = InitialStep.GetID();
MDPState init = new MDPState(initID);
working.Push(new KeyValuePair<MDPConfiguration, MDPState>(InitialStep as MDPConfiguration, init));
MDP mdp = new MDP(init, Ultility.Ultility.DEFAULT_PRECISION, Ultility.Ultility.MAX_DIFFERENCE);
//if initial state is target
ExpressionValue initV = EvaluatorDenotational.Evaluate(ReachableStateCondition, InitialStep.GlobalEnv);
if ((initV as BoolConstant).Value)
{
mdp.AddTargetStates(init);
VerificationOutput.NoOfStates = 1;
return mdp;
}
//if initial state is target
do
{
if (CancelRequested)
{
VerificationOutput.NoOfStates = mdp.States.Count;
return mdp;
}
KeyValuePair<MDPConfiguration, MDPState> current = working.Pop();
IEnumerable<MDPConfiguration> list = current.Key.MakeOneMoveLocal();
VerificationOutput.Transitions += list.Count();
int currentDistriIndex = -1;
Distribution newDis = new Distribution(Constants.TAU);
//for (int i = 0; i < list.Count; i++)
foreach (var step in list)
{
//MDPConfiguration step = list[i];
string stepID = step.GetID();
MDPState nextState;
if (!mdp.States.TryGetValue(stepID, out nextState))
{
nextState = new MDPState(stepID);
mdp.AddState(nextState);
ExpressionValue v = EvaluatorDenotational.Evaluate(ReachableStateCondition, step.GlobalEnv);
if ((v as BoolConstant).Value)
{
mdp.AddTargetStates(nextState);
}
else
{
working.Push(new KeyValuePair<MDPConfiguration, MDPState>(step, nextState));
}
}
if (step.DisIndex == -1)
{
if (currentDistriIndex >= 0)
{
current.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
}
Distribution newTrivialDis = new Distribution(step.Event);
newTrivialDis.AddProbStatePair(1, nextState);
current.Value.AddDistribution(newTrivialDis);
}
else if (currentDistriIndex != -1 && step.DisIndex != currentDistriIndex)
{
current.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
newDis.AddProbStatePair(step.Probability, nextState);
}
else
{
newDis.AddProbStatePair(step.Probability, nextState);
}
currentDistriIndex = step.DisIndex;
}
if (currentDistriIndex >= 0)
{
current.Value.AddDistribution(newDis);
}
} while (working.Count > 0);
VerificationOutput.NoOfStates = mdp.States.Count;
//mdp.BackUpTargetStates();
return mdp;
}
public TupleSub(MDPConfiguration imp, DeterministicFAState_Subset spec, MDPState mdp)
{
ImplState = imp;
SpecState = spec;
MDPState = mdp;
}
private MDP BuildMDPSubset()
{
DeterministicAutomata_Subset specAutomaton = BuildDeterministicAutomata_Subset(InitSpecStep);
Stack<TupleSub> working = new Stack<TupleSub>(1024);
DeterministicFAState_Subset InitialSpec = specAutomaton.InitialState;
Dictionary<string, Dictionary<int, MDPState>> visited = new Dictionary<string, Dictionary<int, MDPState>>();
int specID = InitialSpec.GetID();
string impID = InitialStep.GetID();
MDPState init = new MDPState(impID + Constants.SEPARATOR + specID);
working.Push(new TupleSub(InitialStep as MDPConfiguration, InitialSpec, init));
Dictionary<int, MDPState> initialDic = new Dictionary<int, MDPState>();
initialDic.Add(specID, init);
visited.Add(impID, initialDic);
MDP mdp = new MDP(init, Ultility.Ultility.DEFAULT_PRECISION, Ultility.Ultility.MAX_DIFFERENCE);
do
{
if (CancelRequested)
{
VerificationOutput.NoOfStates = mdp.States.Count;
return mdp;
}
TupleSub current = working.Pop();
if (current.SpecState == null)
{
mdp.AddTargetStates(current.MDPState);
}
else
{
IEnumerable<MDPConfiguration> list = current.ImplState.MakeOneMoveLocal();
VerificationOutput.Transitions += list.Count();
int currentDistriIndex = -1;
Distribution newDis = new Distribution(Constants.TAU);
//for (int i = 0; i < list.Count; i++)
foreach (var step in list)
{
//MDPConfiguration step = list[i];
DeterministicFAState_Subset nextSpec = current.SpecState;
if (step.Event != Constants.TAU)
{
nextSpec = current.SpecState.Next(step.Event);
}
//System.Diagnostics.Debug.Assert(nextSpec != null, "NextSpec is null!!");
//string stepID = step.GetID() + Constants.SEPARATOR + nextSpec.GetID().ToString();
impID = step.GetID();
string stepID = impID + Constants.SEPARATOR + (nextSpec == null ? "" : nextSpec.GetID().ToString());
MDPState nextState;
if (!mdp.States.TryGetValue(stepID, out nextState))
{
nextState = new MDPState(stepID);
//mdp.AddState(nextState);
//KeyValuePair<MDPConfiguration, DeterministicFAState_Subset> newPair =
// new KeyValuePair<MDPConfiguration, DeterministicFAState_Subset>(step, nextSpec);
working.Push(new TupleSub(step, nextSpec, nextState));
if (nextSpec != null)
{
specID = nextSpec.GetID();
//DeterministicFAState_Subset DFAstate = nextSpec;
Dictionary<int, MDPState> mapping = null;
if (visited.TryGetValue(impID, out mapping))
{
//int Spec = nextSpec.GetID();
mapping.Add(specID, nextState);
foreach (int spec in mapping.Keys)
{
if (specAutomaton.States[spec].Sub.Contains(nextSpec))
{
mapping[spec].Sub.Add(nextState);
}
else if (nextSpec.Sub.Contains(specAutomaton.States[spec]))
{
nextState.Sub.Add(mapping[spec]);
}
}
//foreach (var x in visited)
//{
// if (x.Key.Key.GetID() == newPair.Key.GetID() &&
// x.SpecState.Sub.Contains(newPair.Value))
// {
// x.Value.Sub.Add(nextState);
// }
// else if (x.Key.Key.GetID() == newPair.Key.GetID() &&
// newPair.Value.Sub.Contains(x.SpecState))
// {
// nextState.Sub.Add(x.Value);
// }
//}
//visited.Add(newPair, nextState);
}
else
{
Dictionary<int, MDPState> newDicDic = new Dictionary<int, MDPState>();
newDicDic.Add(specID, nextState);
visited.Add(impID, newDicDic);
}
}
mdp.AddState(nextState);
}
if (step.DisIndex == -1)
{
if (currentDistriIndex >= 0)
{
current.MDPState.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
}
Distribution newTrivialDis = new Distribution(step.Event, nextState);
current.MDPState.AddDistribution(newTrivialDis);
}
else if (currentDistriIndex != -1 && step.DisIndex != currentDistriIndex)
{
current.MDPState.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
newDis.AddProbStatePair(step.Probability, nextState);
}
else
{
newDis.AddProbStatePair(step.Probability, nextState);
}
currentDistriIndex = step.DisIndex;
}
if (currentDistriIndex >= 0)
{
current.MDPState.AddDistribution(newDis);
}
}
} while (working.Count > 0);
VerificationOutput.NoOfStates = mdp.States.Count;
//mdp.BackUpTargetStates();
return mdp;
}
private MDP BuildMDPAntiChain_S()
{
DeterministicAutomata_Subset specAutomaton = BuildDeterministicAutomata_Subset(InitSpecStep);
Stack<TupleSub> working = new Stack<TupleSub>(1024);
DeterministicFAState_Subset InitialSpec = specAutomaton.InitialState;
Dictionary<string, HashSet<DeterministicFAState_Subset>> antichain = new Dictionary<string, HashSet<DeterministicFAState_Subset>>();
string impID = InitialStep.GetID();
int specID = InitialSpec.GetID();
MDPState init = new MDPState(impID + Constants.SEPARATOR + specID);
working.Push(new TupleSub(InitialStep as MDPConfiguration, InitialSpec, init));
MDP mdp = new MDP(init, Ultility.Ultility.DEFAULT_PRECISION, Ultility.Ultility.MAX_DIFFERENCE);
HashSet<DeterministicFAState_Subset> Specs = new HashSet<DeterministicFAState_Subset>();
Specs.Add(InitialSpec);
antichain.Add(impID, Specs);
do
{
if (CancelRequested)
{
VerificationOutput.NoOfStates = mdp.States.Count;
return mdp;
}
TupleSub current = working.Pop();
if (current.SpecState == null)
{
mdp.AddTargetStates(current.MDPState);
}
else
{
IEnumerable<MDPConfiguration> list = current.ImplState.MakeOneMoveLocal();
VerificationOutput.Transitions += list.Count();
int currentDistriIndex = -1;
Distribution newDis = new Distribution(Constants.TAU);
//for (int i = 0; i < list.Count; i++)
foreach (MDPConfiguration step in list)
{
//MDPConfiguration step = list[i];
DeterministicFAState_Subset nextSpec = current.SpecState;
if (step.Event != Constants.TAU)
{
nextSpec = current.SpecState.Next(step.Event);
}
//System.Diagnostics.Debug.Assert(nextSpec != null, "NextSpec is null!!");
//string stepID = step.GetID() + Constants.SEPARATOR + nextSpec.GetID().ToString();
impID = step.GetID();
string stepID = impID + Constants.SEPARATOR + (nextSpec == null ? "" : nextSpec.GetID().ToString());
MDPState nextState;
if (!mdp.States.TryGetValue(stepID, out nextState))
{
nextState = new MDPState(stepID);
//mdp.AddState(nextState);
//KeyValuePair<MDPConfiguration, DeterministicFAState_Subset> newPair =
// new KeyValuePair<MDPConfiguration, DeterministicFAState_Subset>(step, nextSpec);
//working.Push(new TupleSub(step, nextSpec, nextState));
bool addstate = true;
if (nextSpec != null)
{
//specID = nextSpec.GetID();
//DeterministicFAState_Subset DFAstate = nextSpec;
HashSet<DeterministicFAState_Subset> specStates;
//Dictionary<int, MDPState> mapping = null;
if (antichain.TryGetValue(impID, out specStates))
{
HashSet<DeterministicFAState_Subset> toRemove =
new HashSet<DeterministicFAState_Subset>();
//int Spec = nextSpec.GetID();
bool chainIncrease = true;
foreach (DeterministicFAState_Subset specState in specStates)
{
//DeterministicFAState_Subset specState = specStates[j];
if (specState.Sub.Contains(nextSpec))
{
toRemove.Add(specState);
nextSpec.Sub.Add(specState);
chainIncrease = false;
}
if (nextSpec.Sub.Contains(specState))
{
nextSpec = specState;
addstate = false;
chainIncrease = false;
break;
}
}
if (toRemove.Count > 0)
{
foreach (var specState in toRemove)
{
specStates.Remove(specState);
}
specStates.Add(nextSpec); // toRemove;
}
else if (chainIncrease)
{
specStates.Add(nextSpec);
}
}
else
{
HashSet<DeterministicFAState_Subset> newSpecs = new HashSet<DeterministicFAState_Subset>();
newSpecs.Add(nextSpec);
antichain.Add(impID, newSpecs);
}
}
if (addstate)
{
mdp.AddState(nextState);
working.Push(new TupleSub(step, nextSpec, nextState));
}
}
if (step.DisIndex == -1)
{
if (currentDistriIndex >= 0)
{
current.MDPState.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
}
Distribution newTrivialDis = new Distribution(step.Event, nextState);
current.MDPState.AddDistribution(newTrivialDis);
}
else if (currentDistriIndex != -1 && step.DisIndex != currentDistriIndex)
{
current.MDPState.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
newDis.AddProbStatePair(step.Probability, nextState);
}
else
{
newDis.AddProbStatePair(step.Probability, nextState);
}
currentDistriIndex = step.DisIndex;
}
if (currentDistriIndex >= 0)
{
current.MDPState.AddDistribution(newDis);
}
}
} while (working.Count > 0);
VerificationOutput.NoOfStates = mdp.States.Count;
//mdp.BackUpTargetStates();
return mdp;
}
private MDP BuildMDP()
{
DeterministicAutomata specAutomaton = BuildDeterministicAutomata(InitSpecStep);
Stack<Tuple> working = new Stack<Tuple>(1024);
DeterministicFAState currentSpec = specAutomaton.InitialState;
MDPState init = new MDPState(InitialStep.GetID() + Constants.SEPARATOR + currentSpec.GetID());
working.Push(new Tuple(InitialStep as MDPConfiguration, currentSpec, init));
MDP mdp = new MDP(init, Ultility.Ultility.DEFAULT_PRECISION, Ultility.Ultility.MAX_DIFFERENCE);
do
{
if (CancelRequested)
{
VerificationOutput.NoOfStates = mdp.States.Count;
return mdp;
}
// KeyValuePair<KeyValuePair<MDPConfiguration, DeterministicFAState>, MDPState>
Tuple current = working.Pop();
if (current.SpecState == null)
{
mdp.AddTargetStates(current.MDPState);
}
else
{
IEnumerable<MDPConfiguration> list = current.ImplState.MakeOneMoveLocal();
VerificationOutput.Transitions += list.Count();
int currentDistriIndex = -1;
Distribution newDis = new Distribution(Constants.TAU);
//for (int i = 0; i < list.Count; i++)
foreach (var step in list)
{
//MDPConfiguration step = list[i];
DeterministicFAState nextSpec = current.SpecState;
if (step.Event != Constants.TAU)
{
nextSpec = current.SpecState.Next(step.Event);
}
//System.Diagnostics.Debug.Assert(nextSpec != null, "NextSpec is null!!");
//string stepID = step.GetID() + Constants.SEPARATOR + nextSpec.GetID().ToString();
string stepID = step.GetID() + Constants.SEPARATOR + (nextSpec == null ? "" : nextSpec.GetID().ToString());
MDPState nextState;
if (!mdp.States.TryGetValue(stepID, out nextState))
{
nextState = new MDPState(stepID);
mdp.AddState(nextState);
working.Push(new Tuple(step, nextSpec, nextState));
}
if (step.DisIndex == -1)
{
if (currentDistriIndex >= 0)
{
current.MDPState.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
}
Distribution newTrivialDis = new Distribution(step.Event, nextState);
current.MDPState.AddDistribution(newTrivialDis);
}
else if (currentDistriIndex != -1 && step.DisIndex != currentDistriIndex)
{
current.MDPState.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
newDis.AddProbStatePair(step.Probability, nextState);
}
else
{
newDis.AddProbStatePair(step.Probability, nextState);
}
currentDistriIndex = step.DisIndex;
}
if (currentDistriIndex >= 0)
{
current.MDPState.AddDistribution(newDis);
}
}
} while (working.Count > 0);
VerificationOutput.NoOfStates = mdp.States.Count;
//mdp.BackUpTargetStates();
return mdp;
}
//protected QueryConstraintType ConstraintType;
//protected DRA DRA;
//public DRA PositiveDRA;
//public DRA NegativeDRA;
//public BuchiAutomata PositiveBA;
//protected double Min = -1;
//protected double Max = -1;
//protected MDP mdp;
//protected Dictionary<string, int> MDPState2DRAStateMapping;
//private bool HasDeadLock;
////private DefinitionRef Process;
//public const string DUMMY_INIT = "dummy";
//private bool AlmostFair = false;
protected void BuildMD_ImprovedTarjan()
{
//int counter = 0;
MDPState dummyInit = new MDPState(DUMMY_INIT);
mdp = new MDP(dummyInit, Ultility.Ultility.DEFAULT_PRECISION, Ultility.Ultility.MAX_DIFFERENCE);
List<PCSPEventDRAPairMEC> initials = GetInitialPairsEMC(InitialStep as MDPConfiguration);
Stack<KeyValuePair<PCSPEventDRAPairMEC, MDPState>> working = new Stack<KeyValuePair<PCSPEventDRAPairMEC, MDPState>>(1024);
//Stack<PCSPEventDRAPairMEC> stepStack = new Stack<PCSPEventDRAPairMEC>(1024);
List<PCSPEventDRAPairMEC> stepStack = new List<PCSPEventDRAPairMEC>(1024);
MDPState2DRAStateMapping = new Dictionary<string, int>(Common.Classes.Ultility.Ultility.MC_INITIAL_SIZE);
//HashSet<string> MDPStateMapping = new HashSet<string>();
//build the MDP while identifying the SCCs
List<List<string>> SCCs = new List<List<string>>(64);
for (int z = 1; z <= initials.Count; z++)
{
PCSPEventDRAPairMEC initState = initials[z - 1];
string stringID = initState.ID;
MDPState newinit = new MDPState(stringID);
mdp.AddState(newinit);
dummyInit.AddDistribution(new Distribution(stringID, newinit));
stepStack.Add(initState);
//newinit.AddDistribution(new Distribution(Constants.TAU, newinit));
working.Push(new KeyValuePair<PCSPEventDRAPairMEC, MDPState>(initState, newinit));
}
Dictionary<string, int> preorder = new Dictionary<string, int>(Common.Classes.Ultility.Ultility.MC_INITIAL_SIZE);
Dictionary<string, int> lowlink = new Dictionary<string, int>(Common.Classes.Ultility.Ultility.MC_INITIAL_SIZE);
Dictionary<string, int> distrRecord = new Dictionary<string, int>();
int preorderCounter = 0;
Dictionary<string, List<PCSPEventDRAPairMEC>> ExpendedNode = new Dictionary<string, List<PCSPEventDRAPairMEC>>();
do
{
if (CancelRequested)
{
this.VerificationOutput.NoOfStates = this.VerificationOutput.NoOfStates + mdp.States.Count;
return;
}
KeyValuePair<PCSPEventDRAPairMEC, MDPState> pair = working.Peek();
MDPConfiguration evt = pair.Key.configuration;
int DRAState = pair.Key.state;
string currentID = pair.Key.ID;
List<Distribution> outgoing = pair.Value.Distributions;
if (!preorder.ContainsKey(currentID))
{
preorder.Add(currentID, preorderCounter);
distrRecord.Add(currentID, 0);
//stepStack.Add(new PCSPEventDRAPairMEC(currentID + "separate" + 0));
preorderCounter++;
}
bool done = true;
if (ExpendedNode.ContainsKey(currentID))
{
List<PCSPEventDRAPairMEC> list = ExpendedNode[currentID];
//int counter = 0;
if (list.Count > 0)
{
int k = list.Count - 1;
//for (int k = list.Count - 1; k >= 0; k--)
PCSPEventDRAPairMEC step = list[k];
if (step == null)
{
stepStack.Add(new PCSPEventDRAPairMEC(currentID + "separate" + (distrRecord[currentID]++)));
list.RemoveAt(k);
continue;
}
else
{
string stepID = step.ID;
if (!preorder.ContainsKey(stepID))
{
//if (done)
//{
working.Push(new KeyValuePair<PCSPEventDRAPairMEC, MDPState>(step, mdp.States[stepID]));
stepStack.Add(step);
done = false;
list.RemoveAt(k);
//}
}
else
{
//stepStack.Add(step.ID);
stepStack.Add(step);
list.RemoveAt(k);
done = false;
}
}
}
//else
//{
// stepStack.Add(new PCSPEventDRAPairMEC(currentID + "end"));
//}
}
else
{
int currentDistriIndex = -1;
Distribution newDis = new Distribution(Constants.TAU);
IEnumerable<MDPConfiguration> steps = evt.MakeOneMoveLocal();
//int counter = 0;
//NOTE: here we play a trick for deadlock case: if a deadlock exist in the MDP, we will make a
//self loop transition to remove the deadlock. Deadlock is meaningless in MDP.
if (evt.IsDeadLock)
{
List<MDPConfiguration> stepsList = new List<MDPConfiguration>(steps);
stepsList.Add(CreateSelfLoopStep(evt));
steps = stepsList;
HasDeadLock = true;
}
List<PCSPEventDRAPairMEC> product = NextMEC(steps.ToArray(), DRAState);
this.VerificationOutput.Transitions += product.Count;
for (int k = product.Count - 1; k >= 0; k--)
{
PCSPEventDRAPairMEC step = product[k];
string tmp = step.ID;
//int nextIndex = VisitedWithID.Count;
MDPState nextState;
if (mdp.States.TryGetValue(tmp, out nextState))
{
if (!preorder.ContainsKey(tmp))
{
if (done)
{
working.Push(new KeyValuePair<PCSPEventDRAPairMEC, MDPState>(step, nextState));
//stepStack.Add(step.ID);
stepStack.Add(step);
product.RemoveAt(k);
done = false;
}
else
{
product[k] = step;
}
}
else
{
if (done)
{
product.RemoveAt(k);
done = false;
}
}
}
else
{
nextState = new MDPState(tmp);
mdp.States.Add(tmp, nextState);
if (done)
{
working.Push(new KeyValuePair<PCSPEventDRAPairMEC, MDPState>(step, nextState));
//stepStack.Add(step.ID);
stepStack.Add(step);
done = false;
product.RemoveAt(k);
}
else
{
product[k] = step;
}
}
MDPConfiguration pstep = step.configuration;
if (pstep.DisIndex == -1)
{
if (currentDistriIndex >= 0)
{
pair.Value.AddDistribution(newDis);
product.Insert(k+1, null);//separator
newDis = new Distribution(Constants.TAU);
}
Distribution newTrivialDis = new Distribution(pstep.Event, nextState);
pair.Value.AddDistribution(newTrivialDis);
if (k != 0)
{
product.Insert(k, null);//separator
}
}
else if (currentDistriIndex != -1 && pstep.DisIndex != currentDistriIndex)
{
pair.Value.AddDistribution(newDis);
product.Insert(k+1, null);//separator
newDis = new Distribution(Constants.TAU);
newDis.AddProbStatePair(pstep.Probability, nextState);
}
else
{
newDis.AddProbStatePair(pstep.Probability, nextState);
}
currentDistriIndex = pstep.DisIndex;
}
if (currentDistriIndex >= 0)
{
pair.Value.AddDistribution(newDis);
}
ExpendedNode.Add(currentID, product);
}
if (done)
{
int lowlinkV = preorder[currentID];
int preorderV = preorder[currentID];
bool selfLoop = false;
//List<bool> temp = new List<bool>(outgoing.Count);
int length = outgoing.Count;
for (int i = 0; i < length; i++)//Distribution list in outgoing)
{
//temp[i] = true;
Distribution list = outgoing[i];//note the order of distribution is reversed
int templowlinkV = int.MaxValue;
//bool tempUpdate = false;
foreach (KeyValuePair<double, MDPState> state in list.States)
{
string w = state.Value.ID;
if (w == currentID && list.States.Count == 1)
{
selfLoop = true;
}
if (!MDPState2DRAStateMapping.ContainsKey(w))
{
//tempUpdate = true;
if (preorder[w] > preorderV)
{
templowlinkV = Math.Min(templowlinkV, lowlink[w]);
//lowlinkV = Math.Min(lowlinkV, lowlink[w]);
}
else
{
templowlinkV = Math.Min(templowlinkV, preorder[w]);
//lowlinkV = Math.Min(lowlinkV, preorder[w]);
}
}
else
{
templowlinkV = int.MaxValue;
break;
}
}
//if (tempUpdate)
//{
//}
if (templowlinkV == int.MaxValue)
{
//int index = stepStack.IndexOf(currentID + "separate" + i);
//for (int index = stepStack.IndexOf(currentID + "separate" + (i)); index < stepStack.Count; )
//{
// if (stepStack[index] != currentID + "separate" + (i+1).ToString() && stepStack[index] != currentID + "end")
// {
// stepStack.RemoveAt(index);
// }
// else
// {
// break;
// }
//}
bool remove = false;
List<int> toRemove = new List<int>();
for (int index = stepStack.Count - 1; index >= 0; index--)
{
//counter++;
if (stepStack[index].ID == currentID + "separate" + i)
{
remove = true;
}
if (remove)
{
if (stepStack[index].ID != currentID + "separate" + (i - 1) && stepStack[index].ID != currentID)
{
stepStack.RemoveAt(index);
}
else
{
break;
}
}
else
{
if (stepStack[index].ID == currentID)
{
foreach(var ind in toRemove)
{
stepStack.RemoveAt(ind);
}
break;
}
toRemove.Add(index);
}
}
}
else
{
lowlinkV = Math.Min(templowlinkV, lowlinkV);
}
}
lowlink[currentID] = lowlinkV;
working.Pop();
if (lowlinkV == preorderV)
{
List<string> SCC = new List<string>(1024);
SCC.Add(currentID);
MDPState2DRAStateMapping.Add(currentID, DRAState);
//while (stepStack.Count > 0 && preorder[stepStack[stepStack.Count - 1]] > preorderV)
while(stepStack[stepStack.Count - 1].ID != currentID)
{
PCSPEventDRAPairMEC s = stepStack[stepStack.Count - 1];
string sID = s.ID;
stepStack.RemoveAt(stepStack.Count - 1);
if (!sID.Contains("separate") && !SCC.Contains(sID))
{
SCC.Add(sID);
MDPState2DRAStateMapping.Add(sID, s.state);
//MDPState2DRAStateMapping.Add(sID, );
}
}
stepStack.RemoveAt(stepStack.Count - 1);
if (SCC.Count > 1 || selfLoop) //evt.IsDeadLock ||
{
SCCs.Add(SCC);
}
}
//else
//{
// stepStack.Push(pair.Key);
//}
}
} while (working.Count > 0);
Debug.WriteLine(mdp.States.Count);
List<string> EndComponents = new List<string>(SCCs.Count);
int count = DRA.acceptance().size();
int helper = 0;
foreach (List<string> scc in SCCs)
{
//for debug
//List<MDPState> debug = new List<MDPState>();
//List<int> drastates = new List<int>();
//foreach(string state in scc)
//{
// debug.Add(mdp.States[state]);
// drastates.Add(MDPState2DRAStateMapping[state]);
//}
//for debug
//if(sentence.Count == 0)
//{
//}
for (int index = 0; index < count; index++)
{
List<string> newSCC = new List<string>();
List<string> targets = new List<string>();
if (AlmostFair)
{
//bool bottom = true;
////int SCCcount = scc.Count;
////note that as long as one SCC(might not be a real MEC) has a U state, the whole SCC cannot be targets.
////RemoveNonECStates(scc, targets);
//foreach (string i in scc)
//{
// int draState = MDPState2DRAStateMapping[i];
// if (bottom)
// {
// if (DRA.acceptance().isStateInAcceptance_U(index, draState))
// {
// bottom = false;
// }
// }
// newSCC.Add(i);
// if (DRA.acceptance().isStateInAcceptance_L(index, draState))
// {
// targets.Add(i);
// }
//}
//if (bottom)
//{
// if (!BottomECStates(newSCC, targets))
// {
// if (newSCC.Count > 0)
// {
// GroupMEC(newSCC);
// }
// }
// else
// {
// Common.Classes.Ultility.Ultility.AddList(EndComponents, scc);
// }
//}
//else
//{
// RemoveNonECStates(newSCC);
// if (newSCC.Count > 0)
// {
// GroupMEC(newSCC);
// }
//}
}
else
{
foreach (string i in scc)
{
int draState = MDPState2DRAStateMapping[i];
if (!DRA.acceptance().isStateInAcceptance_U(index, draState))
{
newSCC.Add(i);
//note add by ssz
if (DRA.acceptance().isStateInAcceptance_L(index, draState))
{
targets.Add(i);
}
//note add by ssz
}
}
if (targets.Count > 0)
{
RemoveNonECStates(newSCC, targets);
}
if (targets.Count > 0)
{
//List<string> endComponent = TarjanModelChecking(index, newSCC);
Debug.WriteLine(helper++);
//Common.Classes.Ultility.Ultility.AddList(EndComponents, endComponent);
Common.Classes.Ultility.Ultility.AddList(EndComponents, scc);//note here newSCC changed to scc
}
else
{
if (scc.Count > 1)
{
GroupMEC(scc);
}
}
}
}
}
foreach (string s in EndComponents)
{
mdp.AddTargetStates(mdp.States[s]);
}
Debug.WriteLine(mdp.States.Count);
VerificationOutput.NoOfStates = VerificationOutput.NoOfStates + mdp.States.Count;
}
/// <summary>
/// This method builds a MDP, while identifying SCCs.
/// </summary>
protected void BuildMDP_OldTarjan()
{
MDPState dummyInit = new MDPState(DUMMY_INIT);
mdp = new MDP(dummyInit, Ultility.Ultility.DEFAULT_PRECISION, Ultility.Ultility.MAX_DIFFERENCE);
List<PCSPEventDRAPair> initials = GetInitialPairs(InitialStep as MDPConfiguration);
Stack<KeyValuePair<PCSPEventDRAPair, MDPState>> working = new Stack<KeyValuePair<PCSPEventDRAPair, MDPState>>(1024);
Stack<PCSPEventDRAPair> stepStack = new Stack<PCSPEventDRAPair>(1024);
MDPState2DRAStateMapping = new Dictionary<string, int>(Common.Classes.Ultility.Ultility.MC_INITIAL_SIZE);
//build the MDP while identifying the SCCs
List<List<string>> SCCs = new List<List<string>>(64);
for (int z = 1; z <= initials.Count; z++)
{
PCSPEventDRAPair initState = initials[z - 1];
string stringID = initState.GetCompressedState();
MDPState newinit = new MDPState(stringID);
mdp.AddState(newinit);
dummyInit.AddDistribution(new Distribution(stringID, newinit));
//newinit.AddDistribution(new Distribution(Constants.TAU, newinit));
working.Push(new KeyValuePair<PCSPEventDRAPair, MDPState>(initState, newinit));
}
Dictionary<string, int> preorder = new Dictionary<string, int>(Common.Classes.Ultility.Ultility.MC_INITIAL_SIZE);
Dictionary<string, int> lowlink = new Dictionary<string, int>(Common.Classes.Ultility.Ultility.MC_INITIAL_SIZE);
int preorderCounter = 0;
Dictionary<string, List<PCSPEventDRAPair>> ExpendedNode = new Dictionary<string, List<PCSPEventDRAPair>>();
do
{
if (CancelRequested)
{
this.VerificationOutput.NoOfStates = this.VerificationOutput.NoOfStates + mdp.States.Count;
return;
}
KeyValuePair<PCSPEventDRAPair, MDPState> pair = working.Peek();
MDPConfiguration evt = pair.Key.configuration;
int DRAState = pair.Key.state;
string currentID = pair.Key.GetCompressedState();
List<Distribution> outgoing = pair.Value.Distributions;
if (!preorder.ContainsKey(currentID))
{
preorder.Add(currentID, preorderCounter);
preorderCounter++;
}
bool done = true;
if (ExpendedNode.ContainsKey(currentID))
{
List<PCSPEventDRAPair> list = ExpendedNode[currentID];
if (list.Count > 0)
{
for (int k = list.Count - 1; k >= 0; k--)
{
PCSPEventDRAPair step = list[k];
string stepID = step.GetCompressedState();
if (!preorder.ContainsKey(stepID))
{
if (done)
{
working.Push(new KeyValuePair<PCSPEventDRAPair, MDPState>(step, mdp.States[stepID]));
done = false;
list.RemoveAt(k);
}
}
else
{
list.RemoveAt(k);
}
}
}
}
else
{
int currentDistriIndex = -1;
Distribution newDis = new Distribution(Constants.TAU);
MDPConfiguration[] steps = evt.MakeOneMoveLocal().ToArray();
//NOTE: here we play a trick for deadlock case: if a deadlock exist in the MDP, we will make a
//self loop transition to remove the deadlock. Deadlock is meaningless in MDP.
if(evt.IsDeadLock)
{
List<MDPConfiguration> stepsList = new List<MDPConfiguration>(steps);
stepsList.Add(CreateSelfLoopStep(evt));
steps = stepsList.ToArray();
HasDeadLock = true;
}
List<PCSPEventDRAPair> product = Next(steps, DRAState);
this.VerificationOutput.Transitions += product.Count;
for (int k = product.Count - 1; k >= 0; k--)
{
PCSPEventDRAPair step = product[k];
string tmp = step.GetCompressedState();
//int nextIndex = VisitedWithID.Count;
MDPState nextState;
if (mdp.States.TryGetValue(tmp, out nextState))
{
if (!preorder.ContainsKey(tmp))
{
if (done)
{
working.Push(new KeyValuePair<PCSPEventDRAPair, MDPState>(step, nextState));
done = false;
product.RemoveAt(k);
}
else
{
product[k] = step;
}
}
else
{
product.RemoveAt(k);
}
}
else
{
nextState = new MDPState(tmp);
mdp.States.Add(tmp, nextState);
if (done)
{
working.Push(new KeyValuePair<PCSPEventDRAPair, MDPState>(step, nextState));
done = false;
product.RemoveAt(k);
}
else
{
product[k] = step;
}
}
MDPConfiguration pstep = step.configuration;
if (pstep.DisIndex == -1)
{
if (currentDistriIndex >= 0)
{
pair.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
}
//note here changed by ssz
//int draState = MDPState2DRAStateMapping[pair.Value.ID];
if(nextState == pair.Value)
{
for (int index = 0; index < DRA.acceptance().size(); index++)
{
if (DRA.acceptance().isStateInAcceptance_L(index, DRAState))
{
mdp.AddTargetStates(pair.Value);
}
}
}
else
{
Distribution newTrivialDis = new Distribution(pstep.Event, nextState);
pair.Value.AddDistribution(newTrivialDis);
}
//note here changed by ssz
}
else if (currentDistriIndex != -1 && pstep.DisIndex != currentDistriIndex)
{
pair.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
newDis.AddProbStatePair(pstep.Probability, nextState);
}
else
{
newDis.AddProbStatePair(pstep.Probability, nextState);
}
currentDistriIndex = pstep.DisIndex;
}
if (currentDistriIndex >= 0)
{
pair.Value.AddDistribution(newDis);
}
ExpendedNode.Add(currentID, product);
}
if (done)
{
int lowlinkV = preorder[currentID];
int preorderV = preorder[currentID];
bool selfLoop = false;
foreach (Distribution list in outgoing)
{
foreach (KeyValuePair<double, MDPState> state in list.States)
{
string w = state.Value.ID;
if (w == currentID)
{
selfLoop = true;
}
if (!MDPState2DRAStateMapping.ContainsKey(w))
{
if (preorder[w] > preorderV)
{
lowlinkV = Math.Min(lowlinkV, lowlink[w]);
}
else
{
lowlinkV = Math.Min(lowlinkV, preorder[w]);
}
}
}
}
lowlink[currentID] = lowlinkV;
working.Pop();
if (lowlinkV == preorderV)
{
List<string> SCC = new List<string>(1024);
SCC.Add(currentID);
MDPState2DRAStateMapping.Add(currentID, DRAState);
while (stepStack.Count > 0 && preorder[stepStack.Peek().GetCompressedState()] > preorderV)
{
PCSPEventDRAPair s = stepStack.Pop();
string sID = s.GetCompressedState();
SCC.Add(sID);
MDPState2DRAStateMapping.Add(sID, s.state);
}
if (SCC.Count > 1 || selfLoop) //evt.IsDeadLock ||
{
SCCs.Add(SCC);
}
}
else
{
stepStack.Push(pair.Key);
}
}
} while (working.Count > 0);
List<string> EndComponents = new List<string>(SCCs.Count);
int count = DRA.acceptance().size();
int helper = 0;
foreach (List<string> scc in SCCs)
{
//for debug
//List<MDPState> debug = new List<MDPState>();
//List<int> drastates = new List<int>();
//foreach(string state in scc)
//{
// debug.Add(mdp.States[state]);
// drastates.Add(MDPState2DRAStateMapping[state]);
//}
//for debug
for (int index = 0; index < count; index++)
{
List<string> newSCC = new List<string>();
List<string> targets = new List<string>();
if (AlmostFair)
{
bool bottom = true;
//int SCCcount = scc.Count;
//note that as long as one SCC(might not be a real MEC) has a U state, the whole SCC cannot be targets.
//RemoveNonECStates(scc, targets);
foreach (string i in scc)
{
int draState = MDPState2DRAStateMapping[i];
if (bottom)
{
if (DRA.acceptance().isStateInAcceptance_U(index, draState))
{
bottom = false;
}
}
newSCC.Add(i);
if (DRA.acceptance().isStateInAcceptance_L(index, draState))
{
targets.Add(i);
}
}
if(bottom)
{
if (!BottomECStates(newSCC, targets))
{
if (newSCC.Count > 0)
{
GroupMEC(newSCC);
}
}
else
{
Common.Classes.Ultility.Ultility.AddList(EndComponents, scc);
}
}
else
{
RemoveNonECStates(newSCC);
if(newSCC.Count>0)
{
GroupMEC(newSCC);
}
}
}
else
{
foreach (string i in scc)
{
int draState = MDPState2DRAStateMapping[i];
if (!DRA.acceptance().isStateInAcceptance_U(index, draState))
{
newSCC.Add(i);
//note add by ssz
if (DRA.acceptance().isStateInAcceptance_L(index, draState))
{
targets.Add(i);
}
//note add by ssz
}
}
//if (AlmostFair)
//{
// RemoveNonECStates(newSCC, targets);
// if (newSCC.Count > 0)
// {
// GroupMEC(newSCC);
// }
// if (newSCC.Count > 0)
// {
// }
//}
//else
//{
if (targets.Count > 0)
{
RemoveNonECStates(newSCC, targets);
}
else
{
continue;
}
if (targets.Count > 0)
{
//List<string> endComponent = TarjanModelChecking(index, newSCC);
Debug.WriteLine(helper++);
//Common.Classes.Ultility.Ultility.AddList(EndComponents, endComponent);
Common.Classes.Ultility.Ultility.AddList(EndComponents, newSCC);
}
//}
}
}
}
foreach (string s in EndComponents)
{
mdp.AddTargetStates(mdp.States[s]);
}
VerificationOutput.NoOfStates = VerificationOutput.NoOfStates + mdp.States.Count;
//mdp.BackUpTargetStates();
}
protected MDP BuildMDP()
{
Stack <KeyValuePair <MDPConfiguration, MDPState> > working = new Stack <KeyValuePair <MDPConfiguration, MDPState> >(1024);
string initID = InitialStep.GetID();
MDPState init = new MDPState(initID);
working.Push(new KeyValuePair <MDPConfiguration, MDPState>(InitialStep as MDPConfiguration, init));
MDP mdp = new MDP(init, Ultility.Ultility.DEFAULT_PRECISION, Ultility.Ultility.MAX_DIFFERENCE);
//if initial state is target
ExpressionValue initV = EvaluatorDenotational.Evaluate(ReachableStateCondition, InitialStep.GlobalEnv);
if ((initV as BoolConstant).Value)
{
mdp.AddTargetStates(init);
VerificationOutput.NoOfStates = 1;
return(mdp);
}
//if initial state is target
do
{
if (CancelRequested)
{
VerificationOutput.NoOfStates = mdp.States.Count;
return(mdp);
}
KeyValuePair <MDPConfiguration, MDPState> current = working.Pop();
IEnumerable <MDPConfiguration> list = current.Key.MakeOneMoveLocal();
VerificationOutput.Transitions += list.Count();
int currentDistriIndex = -1;
Distribution newDis = new Distribution(Constants.TAU);
//for (int i = 0; i < list.Count; i++)
foreach (var step in list)
{
//MDPConfiguration step = list[i];
string stepID = step.GetID();
MDPState nextState;
if (!mdp.States.TryGetValue(stepID, out nextState))
{
nextState = new MDPState(stepID);
mdp.AddState(nextState);
ExpressionValue v = EvaluatorDenotational.Evaluate(ReachableStateCondition, step.GlobalEnv);
if ((v as BoolConstant).Value)
{
mdp.AddTargetStates(nextState);
}
else
{
working.Push(new KeyValuePair <MDPConfiguration, MDPState>(step, nextState));
}
}
if (step.DisIndex == -1)
{
if (currentDistriIndex >= 0)
{
current.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
}
Distribution newTrivialDis = new Distribution(step.Event);
newTrivialDis.AddProbStatePair(1, nextState);
current.Value.AddDistribution(newTrivialDis);
}
else if (currentDistriIndex != -1 && step.DisIndex != currentDistriIndex)
{
current.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
newDis.AddProbStatePair(step.Probability, nextState);
}
else
{
newDis.AddProbStatePair(step.Probability, nextState);
}
currentDistriIndex = step.DisIndex;
}
if (currentDistriIndex >= 0)
{
current.Value.AddDistribution(newDis);
}
} while (working.Count > 0);
VerificationOutput.NoOfStates = mdp.States.Count;
//mdp.BackUpTargetStates();
return(mdp);
}
void Take_Action()
{
//towers cost 100 gold, cant place if too poor
if (coins < 100)
{
return;
}
//loop through all possible actions
List <double> movePerc = new List <double>
{
.4, //left
.4, //up
.2 //right
};
List <Vector2> movements = new List <Vector2>
{
new Vector2(-speed, 0), //left
new Vector2(0, speed), //up
new Vector2(speed, 0) //right
};
int current_state_index = 0;
List <int> indx = new List <int>(new int[groundMinions.Count]);
List <List <Vector2> > allGroundMinionLists = new List <List <Vector2> >();
List <double> groundProbs = new List <double>();
while (groundMinions.Count > 0 && indx[0] < 3)
{
List <Vector2> temp = new List <Vector2>(groundMinions);
groundProbs.Add(0);
for (int i = 0; i < groundMinions.Count; i++)
{
temp[i] = temp[i] + movements[indx[i]];
if (groundProbs[current_state_index] == 0)
{
groundProbs[current_state_index] = movePerc[indx[i]];
}
else
{
groundProbs[current_state_index] *= movePerc[indx[i]];
}
}
int curr_ind = groundMinions.Count - 1;
while (indx[curr_ind] > 1 && curr_ind > 0)
{
indx[curr_ind] = 0;
curr_ind--;
}
indx[curr_ind]++;
allGroundMinionLists.Add(temp);
}
current_state_index = 0;
indx = new List <int>(new int[airMinions.Count]);
List <List <Vector2> > allAirMinionLists = new List <List <Vector2> >();
List <double> airProbs = new List <double>();
while (airMinions.Count > 0 && indx[0] < 3)
{
List <Vector2> temp = new List <Vector2>(airMinions);
airProbs.Add(0);
for (int i = 0; i < airMinions.Count; i++)
{
temp[i] = temp[i] + movements[indx[i]];
if (airProbs[current_state_index] == 0)
{
airProbs[current_state_index] = movePerc[indx[i]];
}
else
{
airProbs[current_state_index] *= movePerc[indx[i]];
}
}
int curr_ind = airMinions.Count - 1;
while (indx[curr_ind] > 1 && curr_ind > 0)
{
indx[curr_ind] = 0;
curr_ind--;
}
indx[curr_ind]++;
allAirMinionLists.Add(temp);
}
int maxReward = 0;
Vector2 bestAction = new Vector2();
int bestType = -1;//0 = ground, 1 = air
foreach (Vector2 tower_loc in possibleTowers)
{
if (!groundTowers.Contains(tower_loc) && !airTowers.Contains(tower_loc))
{
List <Vector2> newGroundTowers = new List <Vector2>(groundTowers);
newGroundTowers.Add(tower_loc);
if (allGroundMinionLists.Count > 0)
{
foreach (List <Vector2> g_minions in allGroundMinionLists)
{
if (allAirMinionLists.Count > 0)
{
foreach (List <Vector2> a_minions in allAirMinionLists)
{
MDPState state = new MDPState(health, coins + 10, newGroundTowers, airTowers, g_minions, a_minions);
int reward = GetReward(state);
if (reward > maxReward)
{
maxReward = reward;
bestAction = tower_loc;
bestType = 0;
}
}
}
else
{
MDPState state = new MDPState(health, coins + 10, newGroundTowers, airTowers, g_minions, new List <Vector2>());
int reward = GetReward(state);
if (reward > maxReward)
{
maxReward = reward;
bestAction = tower_loc;
bestType = 0;
}
}
}
}
else
{
foreach (List <Vector2> a_minions in allAirMinionLists)
{
MDPState state = new MDPState(health, coins + 10, newGroundTowers, airTowers, new List <Vector2>(), a_minions);
int reward = GetReward(state);
if (reward > maxReward)
{
maxReward = reward;
bestAction = tower_loc;
bestType = 0;
}
}
}
}
if (!airTowers.Contains(tower_loc) && !groundTowers.Contains(tower_loc))
{
List <Vector2> newAirTowers = new List <Vector2>(airTowers);
newAirTowers.Add(tower_loc);
if (allGroundMinionLists.Count > 0)
{
foreach (List <Vector2> g_minions in allGroundMinionLists)
{
if (allAirMinionLists.Count > 0)
{
foreach (List <Vector2> a_minions in allAirMinionLists)
{
MDPState state = new MDPState(health, coins + 10, groundTowers, newAirTowers, g_minions, a_minions);
int reward = GetReward(state);
if (reward > maxReward)
{
maxReward = reward;
bestAction = tower_loc;
bestType = 1;
}
}
}
else
{
MDPState state = new MDPState(health, coins + 10, groundTowers, newAirTowers, g_minions, new List <Vector2>());
int reward = GetReward(state);
if (reward > maxReward)
{
maxReward = reward;
bestAction = tower_loc;
bestType = 1;
}
}
}
}
else
{
foreach (List <Vector2> a_minions in allAirMinionLists)
{
MDPState state = new MDPState(health, coins + 10, groundTowers, newAirTowers, new List <Vector2>(), a_minions);
int reward = GetReward(state);
if (reward > maxReward)
{
maxReward = reward;
bestAction = tower_loc;
bestType = 1;
}
}
}
}
}
//now take that action
if (bestType == 0)
{
Instantiate(GT, bestAction, new Quaternion(0, 0, 0, 0));
coins -= 100;
groundTowers.Add(bestAction);
}
else if (bestType == 1)
{
Instantiate(AT, bestAction, new Quaternion(0, 0, 0, 0));
coins -= 100;
airTowers.Add(bestAction);
}
else
{
print("ERROR: NO ACTIONS AVAILABLE");
}
}
private MDP BuildMDP()
{
Stack<KeyValuePair<MDPConfiguration, MDPState>> working = new Stack<KeyValuePair<MDPConfiguration, MDPState>>(1024);
string initID = InitialStep.GetID();
MDPState init = new MDPState(initID);
working.Push(new KeyValuePair<MDPConfiguration, MDPState>(InitialStep as MDPConfiguration, init));
MDP mdp = new MDP(init, Ultility.Ultility.DEFAULT_PRECISION, Ultility.Ultility.MAX_DIFFERENCE);
do
{
if (CancelRequested)
{
VerificationOutput.NoOfStates = mdp.States.Count;
return mdp;
}
KeyValuePair<MDPConfiguration, MDPState> current = working.Pop();
IEnumerable<MDPConfiguration> list = current.Key.MakeOneMoveLocal();
VerificationOutput.Transitions += list.Count();
//check if it is one of the target state
//if (list.Length == 0)
if(current.Key.IsDeadLock)
{
if (isNotTerminationTesting || current.Key.Event != Constants.TERMINATION)
{
mdp.AddTargetStates(current.Value);
}
}
else
{
int currentDistriIndex = -1;
Distribution newDis = new Distribution(Constants.TAU);
//for (int i = 0; i < list.Length; i++)
foreach (var step in list)
{
//MDPConfiguration step = list[i];
string stepID = step.GetID();
MDPState nextState;
if (!mdp.States.TryGetValue(stepID, out nextState))
{
nextState = new MDPState(stepID);
mdp.AddState(nextState);
working.Push(new KeyValuePair<MDPConfiguration, MDPState>(step, nextState));
}
if (step.DisIndex == -1)
{
if (currentDistriIndex >= 0)
{
current.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
}
Distribution newTrivialDis = new Distribution(step.Event);
newTrivialDis.AddProbStatePair(1, nextState);
current.Value.AddDistribution(newTrivialDis);
}
else if (currentDistriIndex != -1 && step.DisIndex != currentDistriIndex)
{
current.Value.AddDistribution(newDis);
newDis = new Distribution(Constants.TAU);
newDis.AddProbStatePair(step.Probability, nextState);
}
else
{
newDis.AddProbStatePair(step.Probability, nextState);
}
currentDistriIndex = step.DisIndex;
}
if (currentDistriIndex >= 0)
{
current.Value.AddDistribution(newDis);
}
}
} while (working.Count > 0);
VerificationOutput.NoOfStates = mdp.States.Count;
//mdp.BackUpTargetStates();
return mdp;
}
