private double GetPriority(State currentState, State nextState, AIAction action, double reward)
{
DiscreteActionValueFunction actionValueFunction = ai.ActionValueFunction;
double discount = ai.Discount;
double bestActionValue = ListUtils.GetHighest(
actionValueFunction.GetActionValues(nextState),
(x, y) => x.Value > y.Value).Value;
return(Math.Abs(reward + discount * bestActionValue - actionValueFunction.GetValue(currentState, action)));
}
public ExpectedSarsa(DiscreteEnvironment environment,
ActionPolicy policy,
double stepSize, double discount)
{
this.environment = environment;
this.policy = policy;
this.stepSize = stepSize;
this.discount = discount;
ActionValueFunction = new DiscreteActionValueFunction(environment);
}
private void UpdateActionValueFunction(State state, State nextState, AIAction action, double reward)
{
double value = ActionValueFunction.GetValue(state, action);
double policyActionValueSum = 0;
foreach (var actionValue in ActionValueFunction.GetActionValues(nextState))
{
policyActionValueSum += policy.GetProbabilityOfAction(actionValue.Key, nextState, ActionValueFunction) * ActionValueFunction.GetValue(nextState, actionValue.Key);
}
double newValue = value + stepSize * (reward + discount * policyActionValueSum - ActionValueFunction.GetValue(state, action));
ActionValueFunction = ActionValueFunction.UpdateValue(state, action, newValue);
}
public void Load()
{
try
{
string JsonString = File.ReadAllText("../../Save/WalkerAIBin.json");
var settings = new JsonSerializerSettings()
{
TypeNameHandling = TypeNameHandling.Objects
};
DiscreteActionValueFunction actionValueFunction = JsonConvert.DeserializeObject <DiscreteActionValueFunction>(JsonString, settings);
ai.ActionValueFunction = actionValueFunction;
}
catch (Exception) {
//TODO
}
}
public void Load()
{
string actionValueFile = "../../Save/WalkerAIBin.json";
string modelFile = "../../Save/WalkerModelAIBin.json";
try
{
DiscreteActionValueFunction actionValueFunction = Load <DiscreteActionValueFunction>(actionValueFile);
Model model = Load <Model>(modelFile);
sarsa.ActionValueFunction = actionValueFunction;
ai.ai = sarsa;
ai.model = model;
}
catch (Exception)
{
//TODO
}
}
private void InitState(State currentState)
{
ActionValueFunction = ActionValueFunction.InitActionValues(currentState);
}