private void ConvertRootCid(int?sourceState, long currentCid)
{
var distanceFromRootOfSourceNode = _cidDistanceFromRoot[currentCid];
var sourceNodeMaxDistanceToLeaf = _cidMaxDistanceFromLeaf[currentCid];
var requiredPadding = _maximalDistanceBetweenStates - distanceFromRootOfSourceNode - sourceNodeMaxDistanceToLeaf;
var stateToStartFrom = CreateNewArtificialPaddingStatesForward(sourceState, requiredPadding);
distanceFromRootOfSourceNode += requiredPadding;
_cidDistanceFromRoot[currentCid] = distanceFromRootOfSourceNode;
StartDistributions(stateToStartFrom);
var cge = _nmdp.GetContinuationGraphElement(currentCid);
if (cge.IsChoiceTypeUnsplitOrFinal)
{
// If a state leads directly into a new state, add this state directly.
// Treat this as if there is a non-deterministic split with only one choice.
// Thus, we assume that this node is not the root node and its non-existing
// source has a distance of 0 from the root.
MarkovDecisionProcess.StartWithNewDistribution();
AddDestination(currentCid, distanceFromRootOfSourceNode);
MarkovDecisionProcess.FinishDistribution();
FinishDistributions(stateToStartFrom);
}
else
{
var cgi = _nmdp.GetContinuationGraphInnerNode(currentCid);
if (cge.IsChoiceTypeForward)
{
// This ChoiceType might be created by ForwardUntakenChoicesAtIndex in ChoiceResolver
throw new Exception("Bug: RootCid cannot be a forward node");
}
else if (cge.IsChoiceTypeNondeterministic)
{
for (var i = cgi.FromCid; i <= cgi.ToCid; i++)
{
MarkovDecisionProcess.StartWithNewDistribution();
AddDestination(i, distanceFromRootOfSourceNode);
MarkovDecisionProcess.FinishDistribution();
}
}
else if (cge.IsChoiceTypeProbabilitstic)
{
MarkovDecisionProcess.StartWithNewDistribution();
for (var i = cgi.FromCid; i <= cgi.ToCid; i++)
{
AddDestination(i, distanceFromRootOfSourceNode);
}
MarkovDecisionProcess.FinishDistribution();
}
FinishDistributions(stateToStartFrom);
for (var i = cgi.FromCid; i <= cgi.ToCid; i++)
{
ConvertChildCid(i);
}
}
}
public void ExampleNoChoices()
{
NestedMarkovDecisionProcess = Examples.ExampleNoChoices.Create();
var converter = new NmdpToMdp(NestedMarkovDecisionProcess);
MarkovDecisionProcess = converter.MarkovDecisionProcess;
int initialDistributions;
int initialTransitions;
double initialProbabilities;
CalculateMetricsOfInitialState(out initialDistributions, out initialTransitions, out initialProbabilities);
initialDistributions.ShouldBe(1);
initialTransitions.ShouldBe(1);
initialProbabilities.ShouldBe(1.0, 0.0000001);
int state0Distributions;
int state0Transitions;
double state0Probabilities;
CalculateMetricsOfState(0, out state0Distributions, out state0Transitions, out state0Probabilities);
state0Distributions.ShouldBe(1);
state0Transitions.ShouldBe(1);
state0Probabilities.ShouldBe(1.0, 0.0000001);
}
private void AddPaddingStatesInMdp()
{
foreach (var paddingState in _paddingStates)
{
if (paddingState.Key.IsInitial)
{
MarkovDecisionProcess.StartWithInitialDistributions();
}
else
{
MarkovDecisionProcess.StartWithNewDistributions(paddingState.Key.State);
}
MarkovDecisionProcess.StartWithNewDistribution();
MarkovDecisionProcess.AddTransition(paddingState.Value, 1.0);
MarkovDecisionProcess.FinishDistribution();
if (paddingState.Key.IsInitial)
{
MarkovDecisionProcess.FinishInitialDistributions();
}
else
{
MarkovDecisionProcess.FinishDistributions();
}
}
}
private void CopyStateLabeling()
{
for (var i = 0; i < _nmdp.States; i++)
{
MarkovDecisionProcess.SetStateLabeling(i, _nmdp.StateLabeling[i]);
}
}
public void ExampleTwoStateProbabilisticSplits()
{
NestedMarkovDecisionProcess = NmdpExamples.ExampleTwoStateProbabilisticSplits.Create();
var converter = new NmdpToMdpByNewStates(NestedMarkovDecisionProcess, Output.TextWriterAdapter(), false);
MarkovDecisionProcess = converter.MarkovDecisionProcess;
int initialDistributions;
int initialTransitions;
double initialProbabilities;
CalculateMetricsOfInitialState(out initialDistributions, out initialTransitions, out initialProbabilities);
initialDistributions.ShouldBe(1);
initialTransitions.ShouldBe(1);
initialProbabilities.ShouldBe(1.0, 0.0000001);
int state0Distributions;
int state0Transitions;
double state0Probabilities;
CalculateMetricsOfState(0, out state0Distributions, out state0Transitions, out state0Probabilities);
state0Distributions.ShouldBe(1);
state0Transitions.ShouldBe(2);
state0Probabilities.ShouldBe(1.0, 0.0000001);
}
public void ExampleTwoStateSplitsProbabilisticThenNondeterministic()
{
NestedMarkovDecisionProcess = NmdpExamples.ExampleTwoStateSplitsProbabilisticThenNondeterministic.Create();
var converter = new NmdpToMdpByNewStates(NestedMarkovDecisionProcess, Output.TextWriterAdapter(), true);
MarkovDecisionProcess = converter.MarkovDecisionProcess;
int initialDistributions;
int initialTransitions;
double initialProbabilities;
CalculateMetricsOfInitialState(out initialDistributions, out initialTransitions, out initialProbabilities);
initialDistributions.ShouldBe(1);
initialTransitions.ShouldBe(1);
initialProbabilities.ShouldBe(1.0, 0.0000001);
int state0Distributions;
int state0Transitions;
double state0Probabilities;
CalculateMetricsOfState(0, out state0Distributions, out state0Transitions, out state0Probabilities);
state0Distributions.ShouldBe(1);
state0Transitions.ShouldBe(2);
state0Probabilities.ShouldBe(1.0, 0.0000001);
AssertDistanceIsEqual(converter.FormulaForArtificalState);
}
internal static MarkovDecisionProcess Create()
{
// Just a simple MDP with simple nondeterministic choices
// ⟳0⟶1⟲
var mdp = new MarkovDecisionProcess(ModelCapacityByMemorySize.Tiny);
mdp.StateFormulaLabels = new string[] { Label1Formula.Label, Label2Formula.Label };
mdp.StateRewardRetrieverLabels = new string[] { };
mdp.StartWithInitialDistributions();
mdp.StartWithNewInitialDistribution();
mdp.AddTransitionToInitialDistribution(0, 1.0);
mdp.FinishInitialDistribution();
mdp.FinishInitialDistributions();
mdp.SetStateLabeling(1, new StateFormulaSet(new[] { true, false }));
mdp.StartWithNewDistributions(1);
mdp.StartWithNewDistribution();
mdp.AddTransition(1, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
mdp.SetStateLabeling(0, new StateFormulaSet(new[] { false, true }));
mdp.StartWithNewDistributions(0);
mdp.StartWithNewDistribution();
mdp.AddTransition(1, 1.0);
mdp.FinishDistribution();
mdp.StartWithNewDistribution();
mdp.AddTransition(0, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
return(mdp);
}
public void ExampleTwoStateSplitsProbabilisticThenNondeterministic()
{
NestedMarkovDecisionProcess = NmdpExamples.ExampleTwoStateSplitsProbabilisticThenNondeterministic.Create();
var converter = new NmdpToMdpByFlattening(NestedMarkovDecisionProcess);
MarkovDecisionProcess = converter.MarkovDecisionProcess;
int initialDistributions;
int initialTransitions;
double initialProbabilities;
CalculateMetricsOfInitialState(out initialDistributions, out initialTransitions, out initialProbabilities);
initialDistributions.ShouldBe(1);
initialTransitions.ShouldBe(1);
initialProbabilities.ShouldBe(1.0, 0.0000001);
int state0Distributions;
int state0Transitions;
double state0Probabilities;
CalculateMetricsOfState(0, out state0Distributions, out state0Transitions, out state0Probabilities);
state0Distributions.ShouldBe(2);
state0Transitions.ShouldBe(4);
state0Probabilities.ShouldBe(2.0, 0.0000001);
}
internal static MarkovDecisionProcess Create()
{
// A MDP which was designed to test prob0e
// 4
// ⇅
// 0⟼1↘
// ↘2⟼3⟲
var mdp = new MarkovDecisionProcess(ModelCapacityByMemorySize.Tiny);
mdp.StateFormulaLabels = new string[] { Label1Formula.Label, Label2Formula.Label };
mdp.StateRewardRetrieverLabels = new string[] { };
mdp.StartWithInitialDistributions();
mdp.StartWithNewInitialDistribution();
mdp.AddTransitionToInitialDistribution(0, 1.0);
mdp.FinishInitialDistribution();
mdp.FinishInitialDistributions();
mdp.SetStateLabeling(0, new StateFormulaSet(new[] { false, false }));
mdp.StartWithNewDistributions(0);
mdp.StartWithNewDistribution();
mdp.AddTransition(1, 1.0);
mdp.FinishDistribution();
mdp.StartWithNewDistribution();
mdp.AddTransition(2, 1.0);
mdp.FinishDistribution();
mdp.StartWithNewDistribution();
mdp.AddTransition(4, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
mdp.SetStateLabeling(1, new StateFormulaSet(new[] { false, false }));
mdp.StartWithNewDistributions(1);
mdp.StartWithNewDistribution();
mdp.AddTransition(3, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
mdp.SetStateLabeling(2, new StateFormulaSet(new[] { false, false }));
mdp.StartWithNewDistributions(2);
mdp.StartWithNewDistribution();
mdp.AddTransition(3, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
mdp.SetStateLabeling(3, new StateFormulaSet(new[] { true, false }));
mdp.StartWithNewDistributions(3);
mdp.StartWithNewDistribution();
mdp.AddTransition(3, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
mdp.SetStateLabeling(4, new StateFormulaSet(new[] { false, false }));
mdp.StartWithNewDistributions(4);
mdp.StartWithNewDistribution();
mdp.AddTransition(0, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
return(mdp);
}
private void FinishDistributions(int?stateToStartFrom)
{
if (stateToStartFrom.HasValue)
{
MarkovDecisionProcess.FinishDistributions();
}
else
{
MarkovDecisionProcess.FinishInitialDistributions();
}
}
private void StartDistributions(int?stateToStartFrom)
{
if (stateToStartFrom.HasValue)
{
var mdpState = stateToStartFrom.Value;
MarkovDecisionProcess.StartWithNewDistributions(mdpState);
}
else
{
MarkovDecisionProcess.StartWithInitialDistributions();
}
}
internal static MarkovDecisionProcess Create()
{
// MDP of [Parker02, page 36]
// 0
// ⇅
// 1➞0.6⟼2⟲
// 0.4⟼3⟲
var mdp = new MarkovDecisionProcess(ModelCapacityByMemorySize.Tiny);
mdp.StateFormulaLabels = new string[] { Label1Formula.Label, Label2Formula.Label };
mdp.StateRewardRetrieverLabels = new string[] { };
mdp.StartWithInitialDistributions();
mdp.StartWithNewInitialDistribution();
mdp.AddTransitionToInitialDistribution(0, 1.0);
mdp.FinishInitialDistribution();
mdp.FinishInitialDistributions();
mdp.SetStateLabeling(0, new StateFormulaSet(new[] { false, false }));
mdp.StartWithNewDistributions(0);
mdp.StartWithNewDistribution();
mdp.AddTransition(1, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
mdp.SetStateLabeling(1, new StateFormulaSet(new[] { false, false }));
mdp.StartWithNewDistributions(1);
mdp.StartWithNewDistribution();
mdp.AddTransition(0, 1.0);
mdp.FinishDistribution();
mdp.StartWithNewDistribution();
mdp.AddTransition(2, 0.6);
mdp.AddTransition(3, 0.4);
mdp.FinishDistribution();
mdp.FinishDistributions();
mdp.SetStateLabeling(2, new StateFormulaSet(new[] { true, false }));
mdp.StartWithNewDistributions(2);
mdp.StartWithNewDistribution();
mdp.AddTransition(2, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
mdp.SetStateLabeling(3, new StateFormulaSet(new[] { false, true }));
mdp.StartWithNewDistributions(3);
mdp.StartWithNewDistribution();
mdp.AddTransition(3, 1.0);
mdp.FinishDistribution();
mdp.FinishDistributions();
return(mdp);
}
private void AddDistribution(int distribution)
{
if (_ltmdpContinuationDistributionMapper.IsDistributionEmpty(distribution))
{
return;
}
MarkovDecisionProcess.StartWithNewDistribution();
var enumerator = _ltmdpContinuationDistributionMapper.GetContinuationsOfDistributionEnumerator(distribution);
while (enumerator.MoveNext())
{
var leaf = GetLeafOfCid(enumerator.CurrentContinuationId);
var probability = GetProbabilityOfCid(enumerator.CurrentContinuationId);
MarkovDecisionProcess.AddTransition(leaf.ToState, probability);
}
MarkovDecisionProcess.FinishDistribution();
}
public void ConvertInitialTransitions()
{
var cidOfStateRoot = _nmdp.GetRootContinuationGraphLocationOfInitialState();
Clear(cidOfStateRoot);
_ltmdpContinuationDistributionMapper.AddInitialDistributionAndContinuation(cidOfStateRoot);
UpdateContinuationDistributionMapperAndCollectLeafs(cidOfStateRoot);
MarkovDecisionProcess.StartWithInitialDistributions();
var numberOfDistributions = _ltmdpContinuationDistributionMapper.GetNumbersOfDistributions();
for (var distribution = 0; distribution < numberOfDistributions; distribution++)
{
AddInitialDistribution(distribution);
}
MarkovDecisionProcess.FinishInitialDistributions();
}
public NmdpToMdpByFlattening(NestedMarkovDecisionProcess nmdp)
: base(nmdp)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
Console.Out.WriteLine("Starting to convert Nested Markov Decision Process to Markov Decision Process");
Console.Out.WriteLine($"Nmdp: States {nmdp.States}, ContinuationGraphSize {nmdp.ContinuationGraphSize}");
var modelCapacity = new ModelCapacityByModelSize(nmdp.States, nmdp.ContinuationGraphSize * 8L);
MarkovDecisionProcess = new MarkovDecisionProcess(modelCapacity);
MarkovDecisionProcess.StateFormulaLabels = nmdp.StateFormulaLabels;
CopyStateLabeling();
ConvertInitialTransitions();
ConvertStateTransitions();
stopwatch.Stop();
_nmdp = null;
Console.Out.WriteLine($"Completed transformation in {stopwatch.Elapsed}");
Console.Out.WriteLine($"Mdp: States {MarkovDecisionProcess.States}, Transitions {MarkovDecisionProcess.Transitions}");
}
private void ConvertChildCid(long currentCid)
{
var distanceFromRootOfSourceNode = _cidDistanceFromRoot[currentCid];
var cge = _nmdp.GetContinuationGraphElement(currentCid);
if (cge.IsChoiceTypeUnsplitOrFinal || cge.IsChoiceTypeForward)
{
return;
}
var mdpState = _cidToArtificialStateMapping[currentCid];
MarkovDecisionProcess.StartWithNewDistributions(mdpState);
var cgi = _nmdp.GetContinuationGraphInnerNode(currentCid);
if (cge.IsChoiceTypeNondeterministic)
{
for (var i = cgi.FromCid; i <= cgi.ToCid; i++)
{
MarkovDecisionProcess.StartWithNewDistribution();
AddDestination(i, distanceFromRootOfSourceNode);
MarkovDecisionProcess.FinishDistribution();
}
}
else if (cge.IsChoiceTypeProbabilitstic)
{
MarkovDecisionProcess.StartWithNewDistribution();
for (var i = cgi.FromCid; i <= cgi.ToCid; i++)
{
AddDestination(i, distanceFromRootOfSourceNode);
}
MarkovDecisionProcess.FinishDistribution();
}
MarkovDecisionProcess.FinishDistributions();
for (var i = cgi.FromCid; i <= cgi.ToCid; i++)
{
ConvertChildCid(i);
}
}
private void ConvertStateTransitions()
{
for (var state = 0; state < _nmdp.States; state++)
{
var cidOfStateRoot = _nmdp.GetRootContinuationGraphLocationOfState(state);
Clear(cidOfStateRoot);
_ltmdpContinuationDistributionMapper.AddInitialDistributionAndContinuation(cidOfStateRoot);
UpdateContinuationDistributionMapperAndCollectLeafs(cidOfStateRoot);
MarkovDecisionProcess.StartWithNewDistributions(state);
var numberOfDistributions = _ltmdpContinuationDistributionMapper.GetNumbersOfDistributions();
for (var distribution = 0; distribution < numberOfDistributions; distribution++)
{
AddDistribution(distribution);
}
MarkovDecisionProcess.FinishDistributions();
}
}
public NmdpToMdpByNewStates(NestedMarkovDecisionProcess nmdp, TextWriter output = null, bool makeConstantDistanceBetweenStates = true)
: base(nmdp)
{
_output = output ?? Console.Out;
_makeConstantDistanceBetweenStates = makeConstantDistanceBetweenStates;
var stopwatch = new Stopwatch();
stopwatch.Start();
_output.WriteLine("Starting to convert Nested Markov Decision Process to Markov Decision Process");
_output.WriteLine($"Nmdp: States {nmdp.States}, ContinuationGraphSize {nmdp.ContinuationGraphSize}");
var newNumberOfStates = nmdp.ContinuationGraphSize;
var newNumberOfTransitions = nmdp.ContinuationGraphSize;
if (_makeConstantDistanceBetweenStates)
{
newNumberOfStates = 30 * newNumberOfStates;
newNumberOfTransitions = 30 * newNumberOfTransitions;
}
var modelCapacity = new ModelCapacityByModelSize(newNumberOfStates, newNumberOfTransitions);
MarkovDecisionProcess = new MarkovDecisionProcess(modelCapacity);
_nmdpStates = _nmdp.States;
CalculateMaxDistanceBetweenStates();
CreateArtificalStateFormula();
ConvertInitialTransitions();
ConvertStateTransitions();
AddPaddingStatesInMdp();
SetStateLabelings();
stopwatch.Stop();
_nmdp = null;
_output.WriteLine($"Completed transformation in {stopwatch.Elapsed}");
_output.WriteLine($"Mdp: States {MarkovDecisionProcess.States}, Transitions {MarkovDecisionProcess.Transitions}");
}
private void AddPaddedTransition(int mdpState, double probability, int requiredPadding)
{
var firstStateBeforePadding = CreateNewArtificialPaddingStatesBackward(mdpState, requiredPadding);
MarkovDecisionProcess.AddTransition(firstStateBeforePadding, probability);
}
