protected override void Backpropagate(MCTSNode node, Reward reward)
{
int player;
while (node != null)
{
node.N++;
node.Q += reward.Value;
Action action = node.Action;
node = node.Parent;
if (node != null)
{
player = node.PlayerID;
this.ActionHistory.Add(new Pair <int, GOB.Action>(player, action));
foreach (MCTSNode child in node.ChildNodes)
{
if (this.ActionHistory.Contains(new Pair <int, GOB.Action>(player, child.Action)))
{
child.NRAVE++;
child.QRAVE += reward.Value;
}
}
}
}
}
public GOB.Action Run()
{
//Default actions that should always be taken
Action a = CheckAlwaysBestAction(InitialNodes[0]);
if (a != null)
{
return(a);
}
MCTSNode selectedNode;
Reward reward;
var startTime = Time.realtimeSinceStartup;
this.CurrentIterationsInFrame = 0;
int currentMCTS = 0;
while (CurrentIterations < MaxIterations)
{
if (CurrentIterationsInFrame >= MaxIterationsProcessedPerFrame)
{
TotalProcessingTime += Time.realtimeSinceStartup - startTime;
return(null);
}
selectedNode = Selection(InitialNodes[currentMCTS]);
if (selectedNode == InitialNodes[currentMCTS])
{
break;
}
reward = Playout(selectedNode.State);
Backpropagate(selectedNode, reward);
CurrentIterationsInFrame++;
CurrentIterations++;
currentMCTS++;
if (currentMCTS == NumberOfRuns)
{
currentMCTS = 0;
}
}
BestFirstChild = BestChildFromSeveral(InitialNodes);
MCTSNode child = BestFirstChild;
BestActionSequence.Clear();
while (child != null)
{
BestActionSequence.Add(child.Action);
child = BestChild(child);
}
InProgress = false;
if (BestFirstChild != null)
{
TotalProcessingTime += Time.realtimeSinceStartup - startTime;
return(BestFirstChild.Action);
}
return(null);
}
public static float CalculateDiscontentment(Action action, List<Goal> goals)
{
var discontentment = 0.0f;
var duration = action.GetDuration();
foreach (var goal in goals)
{
var newValue = goal.InsistenceValue + action.GetGoalChange(goal);
newValue += duration*goal.ChangeRate;
if (newValue > 10.0f)
{
newValue = 10.0f;
}
else if (newValue < 0.0f)
{
newValue = 0.0f;
}
discontentment += goal.GetDiscontentment(newValue);
}
return discontentment;
}
//Simulates one or many playouts of an action in a state. Only applies to swordattack as all other actions are not stochastic.
//Afterwards merges them in MergeStates to average out the results.
protected IWorldModel StochasticPlayout(Action action, IWorldModel prevState, int n)
{
if (action.Name.Contains("SwordAttack") && n > 0)
{
IWorldModel[] testStates = new WorldModelFEAR[n];
//IWorldModel[] testStates = new WorldModel[n];
for (int i = 0; i < n; i++)
{
TotalPlayouts++;
testStates[i] = prevState.GenerateChildWorldModel();
action.ApplyActionEffects(testStates[i]);
}
prevState = MergeStates(testStates, (SwordAttack)action);
}
else
{
TotalPlayouts++;
prevState = prevState.GenerateChildWorldModel();
action.ApplyActionEffects(prevState);
}
return(prevState);
}
public virtual Action GetNextAction()
{
Action action = null;
//returns the next action that can be executed or null if no more executable actions exist
if (this.ActionEnumerator.MoveNext())
{
action = this.ActionEnumerator.Current;
}
while (action != null && !action.CanExecute(this))
{
if (this.ActionEnumerator.MoveNext())
{
action = this.ActionEnumerator.Current;
}
else
{
action = null;
}
}
return(action);
}
public abstract WorldModel GenerateChildWorldModel(Action action);