public ExplorerDecision <int> Explore(PRG random, int numActions)
{
return(ExplorerDecision.Create(
random.UniformInt(1, numActions),
new GenericExplorerState {
Probability = 1f
},
shouldRecord: true));
}
public ExplorerDecision <TAction> MapContext(PRG random, IReadOnlyCollection <TAction> policyActions, int numActions)
{
// Invoke the default policy function to get the action
TAction chosenDecision = default(TAction);
float actionProbability = 0f;
if (this.explore)
{
// Select bag
int chosenBag = random.UniformInt(0, policyActions.Count - 1);
int[] actionsSelected = Enumerable.Repeat <int>(0, numActions).ToArray();
int currentBag = 0;
foreach (var policyAction in policyActions)
{
var actionFromBag = this.GetTopAction(policyAction);
if (actionFromBag == 0 || actionFromBag > numActions)
{
throw new ArgumentException("Action chosen by default policy is not within valid range.");
}
//this won't work if actions aren't 0 to Count
actionsSelected[actionFromBag - 1]++; // action id is one-based
if (currentBag == chosenBag)
{
chosenDecision = policyAction;
}
currentBag++;
}
actionProbability = (float)actionsSelected[this.GetTopAction(chosenDecision) - 1] / policyActions.Count; // action id is one-based
}
else
{
chosenDecision = policyActions.First();
actionProbability = 1f;
}
GenericExplorerState explorerState = new GenericExplorerState
{
Probability = actionProbability
};
return(ExplorerDecision.Create(chosenDecision, explorerState, true));
}
public override ExplorerDecision <int> MapContext(PRG random, int policyAction, int numActionsVariable)
{
if (policyAction == 0 || policyAction > numActionsVariable)
{
throw new ArgumentException("Action chosen by default policy is not within valid range.");
}
float actionProbability;
bool isExplore;
float epsilon = explore ? this.defaultEpsilon : 0f;
float baseProbability = epsilon / numActionsVariable; // uniform probability
if (random.UniformUnitInterval() < 1f - epsilon)
{
actionProbability = 1f - epsilon + baseProbability;
isExplore = false;
}
else
{
// Get uniform random 1-based action ID
int actionId = random.UniformInt(1, numActionsVariable);
if (actionId == policyAction)
{
// If it matches the one chosen by the default policy
// then increase the probability
actionProbability = 1f - epsilon + baseProbability;
}
else
{
// Otherwise it's just the uniform probability
actionProbability = baseProbability;
}
policyAction = actionId;
isExplore = true;
}
EpsilonGreedyState explorerState = new EpsilonGreedyState
{
Epsilon = epsilon,
IsExplore = isExplore,
Probability = actionProbability
};
return(ExplorerDecision.Create(policyAction, explorerState, true));
}
public ExplorerDecision <int[]> Explore(PRG random, int numActionsVariable)
{
var ranking = Enumerable.Range(1, numActionsVariable).ToArray();
for (int i = 0; i < ranking.Length - 1 && i < maxPermutations; i++)
{
int swapIndex = random.UniformInt(i, ranking.Length - 1);
int temp = ranking[swapIndex];
ranking[swapIndex] = ranking[i];
ranking[i] = temp;
}
return(ExplorerDecision.Create(ranking, new GenericExplorerState {
Probability = 1f
}, true));
}
public override ExplorerDecision <int> MapContext(PRG random, int policyAction, int numActionsVariable)
{
if (policyAction == 0 || policyAction > numActionsVariable)
{
throw new ArgumentException("Action chosen by default policy is not within valid range.");
}
int chosenAction = 0;
float actionProbability = 0f;
bool shouldRecordDecision = true;
bool isExplore = true;
int tau = this.tau;
lock (this.lockObject)
{
if (this.tau > 0 && this.explore)
{
this.tau--;
chosenAction = random.UniformInt(1, numActionsVariable);
actionProbability = 1f / numActionsVariable;
isExplore = true;
}
else
{
chosenAction = policyAction;
actionProbability = 1f;
isExplore = false;
}
}
TauFirstState explorerState = new TauFirstState
{
IsExplore = isExplore,
Probability = actionProbability,
Tau = tau
};
return(ExplorerDecision.Create(chosenAction, explorerState, shouldRecordDecision));
}
public ExplorerDecision <int[]> MapContext(PRG random, int[] policyAction, int numActions)
{
MultiActionHelper.ValidateActionList(policyAction);
float epsilon = this.explore ? this.defaultEpsilon : 0f;
int[] chosenAction;
bool isExplore;
if (random.UniformUnitInterval() < epsilon)
{
// 1 ... n
chosenAction = Enumerable.Range(1, policyAction.Length).ToArray();
// 0 ... n - 2
for (int i = 0; i < policyAction.Length - 1; i++)
{
int swapIndex = random.UniformInt(i, policyAction.Length - 1);
int temp = chosenAction[swapIndex];
chosenAction[swapIndex] = chosenAction[i];
chosenAction[i] = temp;
}
isExplore = true;
}
else
{
chosenAction = policyAction;
isExplore = false;
}
EpsilonGreedySlateState explorerState = new EpsilonGreedySlateState
{
Epsilon = this.defaultEpsilon,
IsExplore = isExplore,
Ranking = policyAction
};
return(ExplorerDecision.Create(chosenAction, explorerState, true));
}