protected virtual Reward Playout(WorldModel initialPlayoutState)
{
Action[] actions = initialPlayoutState.GetExecutableActions();
float score = 0;
foreach (Action action in actions)
{
WorldModel worldModel = initialPlayoutState.GenerateChildWorldModel();
int depthCount = 0;
while (!worldModel.IsTerminal())
{
Action randomAction = actions[RandomGenerator.Next(actions.Length)];
randomAction.ApplyActionEffects(worldModel);
depthCount++;
}
if (depthCount > MaxPlayoutDepthReached)
{
MaxPlayoutDepthReached = depthCount;
}
score += worldModel.GetScore();
}
return(new Reward()
{
Value = score / actions.Length,
PlayerID = initialPlayoutState.GetNextPlayer(),
});
}
protected virtual Reward Playout(WorldModel initialPlayoutState)
{
GOB.Action action;
GOB.Action[] actions;
Reward reward = new Reward();
WorldModel current = initialPlayoutState;
int random;
actions = current.GetExecutableActions();
if (actions.Length == 0)
{
reward.PlayerID = current.GetNextPlayer();
reward.Value = 0;
}
while (!current.IsTerminal())
{
current = current.GenerateChildWorldModel();
random = RandomGenerator.Next(0, actions.Length);
action = actions[random];
action.ApplyActionEffects(current);
current.CalculateNextPlayer();
}
reward.PlayerID = current.GetNextPlayer();
reward.Value = current.GetScore();
return(reward);
}
protected override Reward Playout(WorldModel initialPlayoutState)
{
WorldModel worldModel = initialPlayoutState.GenerateChildWorldModel();
int depthCount = 0;
while (!worldModel.IsTerminal() && depthCount <= MaxPlayoutDepth)
{
Action[] actions = worldModel.GetExecutableActions();
Action biasedAction = actions.First();
foreach (Action action in actions)
{
if (action.GetHValue(worldModel) < biasedAction.GetHValue(worldModel))
{
biasedAction = action;
}
}
biasedAction.ApplyActionEffects(worldModel);
depthCount++;
}
if (depthCount > MaxPlayoutDepthReached)
{
base.MaxPlayoutDepthReached = depthCount;
}
return(new Reward()
{
Value = GetWorldModelScore(worldModel),
PlayerID = initialPlayoutState.GetNextPlayer(),
});
}
protected override Reward Playout(WorldModel initialPlayoutState)
{
GOB.Action[] actions = initialPlayoutState.GetExecutableActions();
int bestHvalue = int.MaxValue;
int bestActionIndex = -1;
WorldModel currentState = initialPlayoutState;
while (!currentState.IsTerminal())
{
for (int i = 0; i < actions.Length; i++)
{
GOB.Action action = actions[i];
int h = action.getHvalue();
if (h < bestHvalue)
{
bestActionIndex = i;
bestHvalue = h;
}
}
WorldModel childState = initialPlayoutState.GenerateChildWorldModel();
actions[bestActionIndex].ApplyActionEffects(childState);
childState.CalculateNextPlayer();
currentState = childState;
base.CurrentDepth++;
}
Reward r = new Reward();
r.Value = currentState.GetScore();
return(r);
}
protected virtual Reward Playout(WorldModel initialPlayoutState)
{
WorldModel childWorldModel = initialPlayoutState.GenerateChildWorldModel();
GOB.Action[] actions = childWorldModel.GetExecutableActions();
int DepthReached = 0;
while (!childWorldModel.IsTerminal())
{
if (actions.Length > 0)
{
int index = this.RandomGenerator.Next(actions.Length);
GOB.Action a = actions[index];
//GOB.Action a = actions[6];
a.ApplyActionEffects(childWorldModel);
childWorldModel.CalculateNextPlayer();
}
DepthReached++;
}
if (DepthReached > this.MaxPlayoutDepthReached)
{
this.MaxPlayoutDepthReached = DepthReached;
}
Reward reward = new Reward
{
PlayerID = childWorldModel.GetNextPlayer(),
Value = childWorldModel.GetScore()
};
return(reward);
}
public virtual Reward Playout(WorldModel initialPlayoutState)
{
GOB.Action action;
GOB.Action[] actions;
int random;
WorldModel state = initialPlayoutState.GenerateChildWorldModel();
while (!state.IsTerminal())
{
//should choose randomly
actions = state.GetExecutableActions();
if (actions.Length == 0)
{
continue;
}
random = RandomGenerator.Next(0, actions.Length);
action = actions [random];
action.ApplyActionEffects(state);
}
Reward r = new Reward();
r.Value = state.GetScore();
r.PlayerID = state.GetNextPlayer();
return(r);
}
protected virtual Reward Playout(WorldModel initialPlayoutState)
{
GOB.Action action;
WorldModel model = initialPlayoutState.GenerateChildWorldModel();
GOB.Action[] actions;
Reward reward = new Reward();
while (!model.IsTerminal())
{
actions = model.GetExecutableActions();
if (actions.Length == 0)
{
break;
}
action = actions[RandomGenerator.Next(0, actions.Length)];
action.ApplyActionEffects(model);
model.CalculateNextPlayer();
}
reward.PlayerID = model.GetNextPlayer();
reward.Value = model.GetScore();
return(reward);
}
protected virtual Reward Playout(WorldModel initialPlayoutState)
{
WorldModel currState = initialPlayoutState;
while (!currState.IsTerminal())
{
var actions = currState.GetExecutableActions();
if (actions.Length > 0)
{
//currState = currState.GenerateChildWorldModel();
int next = this.RandomGenerator.Next(0, actions.Length);
currState = StochasticPlayout(actions[next], currState);
currState.CalculateNextPlayer();
//actions[next].ApplyActionEffects(currState);
//currState.CalculateNextPlayer();
}
else
{
break;
}
}
return(new Reward
{
PlayerID = currState.GetNextPlayer(),
Value = currState.GetScore()
});
}
protected override float RunPlayout(WorldModel initialPlayoutState)
{
GOB.Action nextAction;
WorldModel currentState = initialPlayoutState;
var currentPlayoutDepth = 0;
while (!currentState.IsTerminal())
{
var executableActions = currentState.GetExecutableActions();
//Bias: Choose among the 50% best
var maxIndex = Convert.ToInt32(Math.Ceiling(executableActions.Count * 0.5));
var orderedExecutableActions = executableActions.OrderByDescending(x => this.Heuristic.H(currentState, x)).Take(maxIndex).ToList();
var index = this.RandomGenerator.Next(0, maxIndex);
nextAction = executableActions[index];
currentState = currentState.GenerateChildWorldModel(nextAction);
currentPlayoutDepth++;
}
if (currentPlayoutDepth > this.MaxPlayoutDepthReached)
{
this.MaxPlayoutDepthReached = currentPlayoutDepth;
}
//var currentPlayer = currentState.GetNextPlayer();
//var value = initialPlayoutState.GetNextPlayer() == currentPlayer ? score : -score;
return(currentState.GetScore());
}
//Rave + Biased
protected override Reward Playout(WorldModel initialPlayoutState)
{
//throw new NotImplementedException();
ActionHistory = new List <Pair <int, GOB.Action> >();
WorldModel childWorldModel = initialPlayoutState.GenerateChildWorldModel();
GOB.Action action;
int playoutReach = 0;
while (!childWorldModel.IsTerminal())
{
//Select a random Action
GOB.Action[] actions = childWorldModel.GetExecutableActions();
double[] actionIndexes = new double[actions.Length];
double heuristicValue = 0.0;
double accumulatedHeuristicValue = 0.0;
double randomIndex;
int chosenActionIndex = 0;
for (int i = 0; i < actions.Length; i++)
{
heuristicValue = actions[i].H(childWorldModel);
accumulatedHeuristicValue += Math.Pow(Math.E, -heuristicValue);
actionIndexes[i] = accumulatedHeuristicValue;
}
randomIndex = this.RandomGenerator.NextDouble() * accumulatedHeuristicValue;
//Debug.Log("Acumulated: " + accumulatedHeuristicValue);
for (int i = 0; i < actions.Length; i++)
{
if (randomIndex <= actionIndexes[i])
{
chosenActionIndex = i;
break;
}
}
ActionHistory.Add(new Pair <int, GOB.Action>(childWorldModel.GetNextPlayer(), actions[chosenActionIndex]));
actions[chosenActionIndex].ApplyActionEffects(childWorldModel);
childWorldModel.CalculateNextPlayer();
playoutReach += 1;
}
if (playoutReach > MaxPlayoutDepthReached)
{
MaxPlayoutDepthReached = playoutReach;
}
Reward reward = new Reward
{
PlayerID = childWorldModel.GetNextPlayer(),
Value = childWorldModel.GetScore()
};
return(reward);
}
private GOB.Action getRandomAction(WorldModel state)
{
//TODO: fix
throw new NotImplementedException();
GOB.Action[] actions = state.GetExecutableActions();
if (actions.Length > 0)
{
return(actions[RandomGenerator.Next() % actions.Length]);
}
return(actions[0]);
}
protected Reward Playout(WorldModel initialPlayoutState)
{
//Action[] actionHistory;
Action a;
Reward reward = new Reward();
System.Random random = new System.Random();
while (!initialPlayoutState.IsTerminal())
{
a = initialPlayoutState.GetExecutableActions()[random.Next()];
this.ActionHistory.Add(new Utils.Pair <int, GOB.Action>(initialPlayoutState.GetNextPlayer(), a));
a.ApplyActionEffects(initialPlayoutState);
}
reward.Value = initialPlayoutState.GetScore();
return(reward);
}
protected override Reward Playout(WorldModel initialPlayoutState)
{
WorldModel childWorldModel = initialPlayoutState.GenerateChildWorldModel();
int DepthReached = 0;
while (!childWorldModel.IsTerminal())
{
GOB.Action[] actions = childWorldModel.GetExecutableActions();
double[] actionIndexes = new double[actions.Length];
double heuristicValue = 0.0;
double accumulatedHeuristicValue = 0.0;
double randomIndex;
int chosenActionIndex = 0;
for (int i = 0; i < actions.Length; i++)
{
heuristicValue = actions[i].H(childWorldModel);
accumulatedHeuristicValue += Math.Pow(Math.E, -heuristicValue);
actionIndexes[i] = accumulatedHeuristicValue;
}
randomIndex = this.RandomGenerator.NextDouble() * accumulatedHeuristicValue;
for (int i = 0; i < actions.Length; i++)
{
if (randomIndex <= actionIndexes[i])
{
chosenActionIndex = i;
break;
}
}
actions[chosenActionIndex].ApplyActionEffects(childWorldModel);
childWorldModel.CalculateNextPlayer();
DepthReached++;
}
if (DepthReached > this.MaxPlayoutDepthReached)
{
this.MaxPlayoutDepthReached = DepthReached;
}
Reward reward = new Reward
{
PlayerID = this.InitialNode.PlayerID,
Value = childWorldModel.GetScore()
};
return(reward);
}
protected virtual Reward Playout(WorldModel initialPlayoutState)
{
WorldModel currentState = initialPlayoutState;
while (!currentState.IsTerminal())
{
GOB.Action[] actions = currentState.GetExecutableActions();
if (actions.Length == 0)
{
continue;
}
int index = this.RandomGenerator.Next(0, actions.Length);
GOB.Action action = actions[index];
currentState = currentState.GenerateChildWorldModel();
action.ApplyActionEffects(currentState);
this.CurrentDepth++;
}
Reward reward = new Reward();
reward.Value = currentState.GetScore();
return(reward);
}
protected override Reward Playout(WorldModel initialPlayoutState)
{
ActionHistory = new List <Pair <int, GOB.Action> >();
WorldModel state = initialPlayoutState.GenerateChildWorldModel();
Action nextAction;
while (!state.IsTerminal())
{
Action[] actions = state.GetExecutableActions();
if (actions.Length > 0)
{
nextAction = actions[RandomGenerator.Next() % actions.Length];
ActionHistory.Add(new Pair <int, GOB.Action>(state.GetNextPlayer(), nextAction));
nextAction.ApplyActionEffects(state);
state.CalculateNextPlayer();
}
}
Reward r = new Reward();
r.Value = state.GetScore();
return(r);
}
public override Reward Playout(WorldModel initialPlayoutState)
{
GOB.Action action = null;
GOB.Action[] actions;
WorldModel state = initialPlayoutState.GenerateChildWorldModel();
while (!state.IsTerminal())
{
//should choose randomly
actions = state.GetExecutableActions();
float best = float.MinValue;
foreach (var a in actions)
{
WorldModel w = state.GenerateChildWorldModel();
a.ApplyActionEffects(w);
var heuristic = w.GetGoalValue("BeQuick") + 1 / w.GetGoalValue("GainXP") + w.GetGoalValue("Survive") + w.GetGoalValue("GetRich");
if (heuristic > best)
{
best = heuristic;
action = a;
}
}
action.ApplyActionEffects(state);
}
Reward r = new Reward();
r.Value = state.GetScore();
r.PlayerID = state.GetNextPlayer();
return(r);
}
protected virtual float RunPlayout(WorldModel currentState)
{
GOB.Action nextAction;
var currentPlayoutDepth = 0;
while (!currentState.IsTerminal())
{
var executableActions = currentState.GetExecutableActions();
nextAction = executableActions[this.RandomGenerator.Next(0, executableActions.Count)];
currentState = currentState.GenerateChildWorldModel(nextAction);
currentPlayoutDepth++;
}
if (currentPlayoutDepth > this.MaxPlayoutDepthReached)
{
this.MaxPlayoutDepthReached = currentPlayoutDepth;
}
//var value = initialPlayoutState.GetNextPlayer() == currentPlayer ? score : -score;
return(currentState.GetScore());
}
protected override Reward Playout(WorldModel initialPlayoutState)
{
GOB.Action action;
GOB.Action[] actions;
Reward reward = new Reward();
WorldModel current = initialPlayoutState;
double random;
float h = 0;
double accumulate = 0;
float euclidean = 0;
double softmax = 0;
List <double> interval = new List <double>();
WalkToTargetAndExecuteAction wa;
actions = current.GetExecutableActions();
if (actions.Length == 0)
{
reward.PlayerID = current.GetNextPlayer();
reward.Value = 0;
}
while (!current.IsTerminal())
{
accumulate = 0;
interval.Clear();
//if (actions.Length == 0)
// break;
foreach (var a in actions)
{
h = 0;
var gameMan = this.CurrentStateWorldModel.GetGameManager();
var character = gameMan.characterData;
wa = a as WalkToTargetAndExecuteAction;
if (wa != null)
{
euclidean = (wa.Target.transform.position - wa.Character.transform.position).magnitude;
if (euclidean <= 0)
{
euclidean = 1;
}
}
if (a.Name.Contains("LevelUp")) //1000
{
h = 1000;
}
if (a.Name.Contains("GetHealthPotion")) //0-25
{
h = (character.MaxHP - character.HP) * 1.5f;
}
else if (a.Name.Contains("PickUpChest")) //5-25
{
h = (character.Money + 5) * 3.5f;
}
else if (a.Name.Contains("FireballSkeleton") || a.Name.Contains("FireballOrc")) //0-25
{
h = character.Mana * 30;
}
else if (a.Name.Contains("SwordAttackSkeleton"))
{
h = (character.HP - 5) * 2;
}
else if (a.Name.Contains("SwordAttackOrc"))
{
h = (character.HP - 10) * 2;
}
else if (a.Name.Contains("SwordAttackDragon"))
{
h = character.HP - 20;
}
if (h < 0)
{
h = 0;
}
h = h * 1000 / euclidean;
accumulate += h;
if (h > 0)
{
softmax += Math.Pow(Math.E, -h / accumulate);
interval.Add(softmax);
Debug.Log(softmax);
}
else
{
interval.Add(0);
}
}
random = RandomGenerator.NextDouble() * softmax;
for (int j = 0; j < interval.Count; j++)
{
if (random <= interval[j])
{
action = actions[j];
current = current.GenerateChildWorldModel();
action.ApplyActionEffects(current);
current.CalculateNextPlayer();
break;
}
if (j == interval.Count - 1)
{
current = current.GenerateChildWorldModel();
reward.Value = 0;
reward.PlayerID = current.GetNextPlayer();
return(reward);
}
}
}
reward.PlayerID = current.GetNextPlayer();
reward.Value = current.GetScore();
return(reward);
}
