/// <summary>
/// THis is implemented for ISupervisedLearingModel so that this model can also be used for TrainerMimic
/// </summary>
/// <param name="vectorObservation"></param>
/// <param name="visualObservation"></param>
/// <returns>(mean, var) var will be null for discrete</returns>
public ValueTuple <float[, ], float[, ]> EvaluateAction(float[,] vectorObservation, List <float[, , , ]> visualObservation, List <float[, ]> actionsMask)
{
List <Array> inputLists = new List <Array>();
if (HasVectorObservation)
{
Debug.Assert(vectorObservation != null, "Must Have vector observation inputs!");
inputLists.Add(vectorObservation);
}
if (HasVisualObservation)
{
Debug.Assert(visualObservation != null, "Must Have visual observation inputs!");
inputLists.AddRange(visualObservation);
}
if (ActionSpace == SpaceType.discrete)
{
int batchSize = vectorObservation != null?vectorObservation.GetLength(0) : visualObservation[0].GetLength(0);
int branchSize = ActionSizes.Length;
List <float[, ]> masks = actionsMask;
//create all 1 mask if the input mask is null.
if (masks == null)
{
masks = CreateDummyMasks(ActionSizes, batchSize);
}
inputLists.AddRange(masks);
}
var result = PolicyFunction.Call(inputLists);
float[,] actions = ((float[, ])result[0].eval());
float[,] outputVar = null;
if (SLHasVar)
{
object varTemp = result[1].eval();
if (varTemp is float[, ])
{
outputVar = (float[, ])result[1].eval();
}
else
{
outputVar = null;
}
}
//normlaized the input observations in every calll of eval action
if (useInputNormalization && HasVectorObservation)
{
UpdateNormalizerFunction.Call(new List <Array>()
{
vectorObservation
});
}
return(ValueTuple.Create(actions, outputVar));
}
/// <summary>
/// Query actions based on curren states. The first dimension of the array must be batch dimension
/// </summary>
/// <param name="vectorObservation">current vector states. Can be batch input</param>
/// <returns></returns>
public virtual float[,] EvaluateAction(float[,] vectorObservation, List <float[, , , ]> visualObservation)
{
List <Array> inputLists = new List <Array>();
if (HasVectorObservation)
{
Debug.Assert(vectorObservation != null, "Must Have vector observation inputs!");
inputLists.Add(vectorObservation);
}
if (HasVisualObservation)
{
Debug.Assert(visualObservation != null, "Must Have visual observation inputs!");
inputLists.AddRange(visualObservation);
}
var result = ActionFunction.Call(inputLists);
var outputAction = ((float[, ])result[0].eval());
float[,] actions = new float[outputAction.GetLength(0), outputAction.GetLength(1)];
actions = outputAction;
if (useInputNormalization && HasVectorObservation)
{
UpdateNormalizerFunction.Call(new List <Array>()
{
vectorObservation
});
}
/*for(int i = 0; i < actions.GetLength(0); ++i)
* {
* for (int j = 0; j < actions.GetLength(1); ++j)
* {
* if (float.IsNaN(actions[i, j]))
* {
* Debug.LogError("error");
* }
* }
* }*/
return(actions);
}
/// <summary>
/// Query actions based on curren states. The first dimension of the array must be batch dimension
/// </summary>
/// <param name="vectorObservation">current vector states. Can be batch input</param>
/// <param name="actionProbs">output actions' probabilities. note that it is the normalized log probability</param>
/// <param name="actoinsMask">action mask for discrete action. </param>
/// <returns></returns>
public virtual float[,] EvaluateAction(float[,] vectorObservation, out float[,] actionProbs, List <float[, , , ]> visualObservation, List <float[, ]> actionsMask = null)
{
Debug.Assert(mode == Mode.PPO, "This method is for PPO mode only");
List <Array> inputLists = new List <Array>();
if (HasVectorObservation)
{
Debug.Assert(vectorObservation != null, "Must Have vector observation inputs!");
inputLists.Add(vectorObservation);
}
if (HasVisualObservation)
{
Debug.Assert(visualObservation != null, "Must Have visual observation inputs!");
inputLists.AddRange(visualObservation);
}
float[,] actions = null;
actionProbs = null;
if (ActionSpace == SpaceType.continuous)
{
var result = ActionFunction.Call(inputLists);
actions = ((float[, ])result[0].eval());
actionProbs = ((float[, ])result[1].eval());
}
else if (ActionSpace == SpaceType.discrete)
{
int batchSize = vectorObservation != null?vectorObservation.GetLength(0) : visualObservation[0].GetLength(0);
int branchSize = ActionSizes.Length;
List <float[, ]> masks = actionsMask;
//create all 1 mask if the input mask is null.
if (masks == null)
{
masks = CreateDummyMasks(ActionSizes, batchSize);
}
inputLists.AddRange(masks);
var result = ActionFunction.Call(inputLists);
actions = ((float[, ])result[0].eval());
//get the log probabilities
actionProbs = new float[batchSize, branchSize];
for (int b = 0; b < branchSize; ++b)
{
var tempProbs = ((float[, ])result[b + 1].eval());
int actSize = ActionSizes[b];
for (int i = 0; i < batchSize; ++i)
{
actionProbs[i, b] = tempProbs[i, Mathf.RoundToInt(actions[i, b])];
}
}
}
//normlaized the input observations in every calll of eval action
if (useInputNormalization && HasVectorObservation)
{
UpdateNormalizerFunction.Call(new List <Array>()
{
vectorObservation
});
}
return(actions);
}
/// <summary>
/// Query actions based on curren states. The first dimension of the array must be batch dimension
/// </summary>
/// <param name="vectorObservation">current vector states. Can be batch input</param>
/// <param name="actionProbs">output actions' probabilities</param>
/// <param name="useProbability">when true, the output actions are sampled based on output mean and variance. Otherwise it uses mean directly.</param>
/// <returns></returns>
public virtual float[,] EvaluateAction(float[,] vectorObservation, out float[,] actionProbs, List <float[, , , ]> visualObservation, bool useProbability = true)
{
Debug.Assert(mode == Mode.PPO, "This method is for PPO mode only");
List <Array> inputLists = new List <Array>();
if (HasVectorObservation)
{
Debug.Assert(vectorObservation != null, "Must Have vector observation inputs!");
inputLists.Add(vectorObservation);
}
if (HasVisualObservation)
{
Debug.Assert(visualObservation != null, "Must Have visual observation inputs!");
inputLists.AddRange(visualObservation);
}
var result = ActionFunction.Call(inputLists);
var outputAction = ((float[, ])result[0].eval());
float[,] actions = new float[outputAction.GetLength(0), ActionSpace == SpaceType.continuous ? outputAction.GetLength(1) : 1];
actionProbs = new float[outputAction.GetLength(0), ActionSpace == SpaceType.continuous ? outputAction.GetLength(1) : 1];
if (ActionSpace == SpaceType.continuous)
{
actions = outputAction;
actionProbs = ((float[, ])result[1].eval());
//var actionsMean = (float[,])(result[2].eval());
//var actionsVars = (float[])(result[3].eval());
//print("actual vars" + actions.GetColumn(0).Variance()+"," + actions.GetColumn(1).Variance() + "," + actions.GetColumn(2).Variance() + "," + actions.GetColumn(3).Variance());
}
else if (ActionSpace == SpaceType.discrete)
{
for (int j = 0; j < outputAction.GetLength(0); ++j)
{
if (useProbability)
{
actions[j, 0] = MathUtils.IndexByChance(outputAction.GetRow(j));
}
else
{
actions[j, 0] = outputAction.GetRow(j).ArgMax();
}
actionProbs[j, 0] = outputAction.GetRow(j)[Mathf.RoundToInt(actions[j, 0])];
}
}
if (useInputNormalization && HasVectorObservation)
{
UpdateNormalizerFunction.Call(new List <Array>()
{
vectorObservation
});
//var runningMean = (float[])runningData[0].eval();
//var runningVar = (float[])runningData[1].eval();
//var steps = (float)runningData[2].eval();
//var normalized = (float[,])runningData[3].eval();
}
return(actions);
}
