/// <summary>
/// Adding all scaled eligibility values to the QTable.
/// Needed for QLearning with eligibility traces.
/// </summary>
/// <param name="scale">The scale to apply to all eligibility traces before adding them.</param>
/// <param name="eTable">The <see cref="T:EligibilityTable"/> containing all eligibility values.</param>
public void AddScaledValues(float scale, EligibilityTable eTable)
{
var qTableCopy = new Dictionary <StateActionPair, float>(_qTable);
foreach (var qValPair in _qTable)
{
qTableCopy[qValPair.Key] = qValPair.Value + eTable.GetEligibilityValue(qValPair.Key) * scale;
}
_qTable = qTableCopy;
}
/// <summary>
/// Constructor of the QLearner
/// </summary>
/// <param name="stateExtractor"></param>
/// The mower, that is learning
/// <param name="mower"></param>
///
/// The following are all classic QLearning parameters,
/// with values in between 0 and 1:
///
/// Greediness of the Q learning algorithm
/// <param name="greediness"></param>
/// Discountvalue: How much are old experiences weighted
/// <param name="discountValue"></param>
/// How fast or slow shall the mower learn?
/// Fast: High value, slow: low value
/// <param name="learnRate"></param>
///
/// "Special" parameters:
///
/// QLearning with optimitstic initial values
/// <param name="initialQValue"></param>
/// File to load the qTable from. Can be null.
/// <param name="qTableDeserializationFile">
/// File to write the qTable to after the episode limit is reached.
/// <param name="qTableSerializationFile">
/// Running with ETraces, yes or no?
/// <param name="eligTraces"></param>
/// If so, with what gamma?
/// <param name="gamma"></param>
/// File to load the EligibilityTable from. Can be null.
/// <param name="eTableDeserializationFile">
/// File to write the EligibilityTable to after the episode limit is reached.
/// <param name="eTableSerializationFile">
/// Running with Model?
/// <param name="modelPlanning"></param>
/// Using refined (true) or DynaQ (false) model
/// <param name="refined"></param>
/// File to load the Model from. Can be null.
/// <param name="modelTableDeserializationFile">
/// File to write the Model to after the episode limit is reached.
/// <param name="modelTableSerializationFile">
/// How many N virtual steps are executed in the model?
/// <param name="n"></param>
/// Shall the parameter label be shown?
/// <param name="showParamLabel"></param>
public QLearner(StateExtractor stateExtractor,
Mower mower,
float greediness,
float discountValue,
float learnRate,
float initialQValue,
string qTableDeserializationFile,
string qTableSerializationFile,
bool eligTraces,
float gamma,
string eTableDeserializationFile,
string eTableSerializationFile,
bool modelPlanning,
bool refined,
string modelTableDeserializationFile,
string modelTableSerializationFile,
int n,
bool showParamLabel)
: base(stateExtractor)
{
Greediness = greediness;
DiscountValue = discountValue;
LearnRate = learnRate;
InitialQValue = initialQValue;
_qTableSerializationFile = qTableSerializationFile;
_qTable = new QTable(InitialQValue, qTableDeserializationFile);
Run_with_etraces = eligTraces;
Gamma = gamma;
_eTableSerializationFile = eTableSerializationFile;
ModelPlanning = modelPlanning;
Refined = refined;
_modelTableSerializationFile = modelTableSerializationFile;
if (Run_with_etraces)
{
_eTable = new EligibilityTable(0f, eTableDeserializationFile);
}
if (ModelPlanning)
{
if (refined)
{
_mTable = new ModelTable(mower, modelTableDeserializationFile);
}
else
{
_simpleMTable = new ModelTableSimple(0f, modelTableDeserializationFile);
}
}
N = n;
_showParamLabel = showParamLabel;
Debug.Log(string.Format(
"QLearner instantiated with greediness:{0}, discountValue:{1}, learnRate:{2}, initialQValue:{3}, N for Model:{4}",
Greediness,
DiscountValue,
LearnRate,
InitialQValue,
N));
EpisodeManager.AddEpisodeChangesReceiver(this);
}
