protected override void AgentReplay(
int batchSize,
OptimizerBase optimizer,
MetricFunction lossMetric,
bool shuffle)
{
var batch = shuffle ? ReplayMemory.ToShuffledBatch(batchSize) : ReplayMemory.ToRandomBatch(batchSize);
var states = new DataFrame <float>(StateShape);
var statesTarget = new DataFrame <float>(StateShape);
foreach (var sample in batch)
{
states.Add(sample.Item1);
statesTarget.Add(sample.Item4 ?? new float[StateShape.TotalSize]);
}
var prediction = Model.Predict(states);
var predictionOfTargetStates = Model.Predict(statesTarget);
var predictionTarget = TargetModel.Predict(statesTarget);
var data = new DataFrameList <float>(StateShape, ActionShape);
for (var i = 0; i < batch.Length; i++)
{
var sample = batch[i];
var t = prediction[i];
if (sample.Item4 == null)
{
t[sample.Item2] = sample.Item3;
}
else
{
var lastValue = float.MinValue;
var valueIndex = 0;
for (var j = 0; j < predictionOfTargetStates[i].Length; j++)
{
if (predictionOfTargetStates[i][j] > lastValue)
{
lastValue = predictionOfTargetStates[i][j];
valueIndex = j;
}
}
t[sample.Item2] = (float)(sample.Item3 + DiscountFactor * predictionTarget[i][valueIndex]);
}
data.AddFrame(sample.Item1, t);
}
Model.Fit(data, 1, batch.Length, optimizer, lossMetric);
}
public static string GetStringValue(this MetricFunction metricFunction)
{
switch (metricFunction)
{
case MetricFunction.Min:
return("min");
case MetricFunction.Max:
return("max");
case MetricFunction.Median:
return("median");
case MetricFunction.HighMedian:
return("high_median");
case MetricFunction.LowMedian:
return("low_median");
case MetricFunction.Mean:
return("mean");
case MetricFunction.HighMean:
return("high_mean");
case MetricFunction.LowMean:
return("low_mean");
case MetricFunction.Metric:
return("metric");
case MetricFunction.Varp:
return("varp");
case MetricFunction.HighVarp:
return("high_varp");
case MetricFunction.LowVarp:
return("low_varp");
default:
throw new ArgumentOutOfRangeException(nameof(metricFunction), metricFunction, null);
}
}
protected MetricDetectorBase(MetricFunction function) : base(function.GetStringValue())
{
}
public MetricDetectorDescriptor(MetricFunction function) : base(function.GetStringValue())
{
}
protected override void BeginProcessing()
{
if (Context == null)
{
Context = Context.CurrentContext;
}
var stopWatch = new Stopwatch();
stopWatch.Start();
for (var epoch = 1; epoch <= MaxEpoch; ++epoch)
{
TrainingData.Reset();
var totalLoss = 0.0f;
var dataSize = 0;
while (!TrainingData.End())
{
var batch = TrainingData.Next();
var data = batch.Data[0].AsInContext(Context);
var label = batch.Label[0].AsInContext(Context);
using (Autograd.Record())
{
var output = Model.Call(data);
var loss = (NDArray)LossFunction.Call(output, label);
loss.Backward();
totalLoss += loss.Sum();
dataSize += data.Shape[0];
Trainer.Step(batch.Data[0].Shape[0]);
}
}
if (epoch % DisplayStep == 0 || epoch == MaxEpoch)
{
totalLoss /= dataSize;
ValidationData.Reset();
var totalValidLoss = 0.0f;
var validDataSize = 0;
if (MetricFunction != null)
{
MetricFunction.Reset();
}
while (!ValidationData.End())
{
var batch = ValidationData.Next();
var data = batch.Data[0].AsInContext(Context);
var label = batch.Label[0].AsInContext(Context);
var output = Model.Call(data);
var validLoss = (NDArray)LossFunction.Call(output, label);
totalValidLoss += validLoss.Sum();
validDataSize += data.Shape[0];
if (MetricFunction != null)
{
MetricFunction.Update(label, output);
}
}
totalValidLoss /= validDataSize;
string metricName = null;
float metric = float.NaN;
if (MetricFunction != null)
{
(metricName, metric) = MetricFunction.Get();
}
var status = new TrainingStatus(epoch,
(float)Math.Round(totalLoss, DisplayDigits),
(float)Math.Round(totalValidLoss, DisplayDigits),
metricName,
(float)Math.Round(metric, DisplayDigits),
stopWatch.Elapsed);
WriteObject(status);
}
}
}
