public static double Evaluate(Learner learner, SequenceData testSequences, double[] solutionData) {
double[] guessedProbs = new double[testSequences.Count];
for (int i = 0; i < guessedProbs.Length; i++)
guessedProbs[i] = learner.CalculateProbability(testSequences[i]);
guessedProbs = Normalize(guessedProbs);
return Evaluate(guessedProbs, solutionData);
}
public IEnumerable<double> RunLearner(DataSet dataSet, Learner learner, StreamWriter outputWriter, StreamWriter csvWriter, int iteration)
{
double[] runScores = new double[numberOfRuns];
double[] runTimes = new double[numberOfRuns];
double[] runTicks = new double[numberOfRuns];
Stopwatch watch = new Stopwatch();
for (int i = 0; i < numberOfRuns; i++)
{
Console.WriteLine("Run {0}...", (i + 1));
watch.Reset();
dataSet.SplitData(2.0 / 3.0, i);
watch.Start();
learner.Learn(dataSet.TrainingData, dataSet.ValidationData, dataSet.TestData);
watch.Stop();
double score = 0.0;
if (useTestData)
{
score = PautomacEvaluator.Evaluate(learner, dataSet.TestData, dataSet.SolutionData);
}
else
{
foreach (int[] signal in dataSet.ValidationData.GetAll())
{
score -= learner.CalculateProbability(signal, true);
}
}
runScores[i] = score;
runTimes[i] = (watch.ElapsedMilliseconds / 1000.0);
runTicks[i] = watch.ElapsedTicks;
outputWriter.WriteLine("Run {0:00}:\t{1:00000000.0000000000}\t{2:000000}\t{3:0000000000000000}", (i + 1), runScores[i], runTimes[i], runTicks[i]);
outputWriter.Flush();
csvWriter.WriteLine("{0},{1},{2},{3},{4},", iteration, i, runScores[i], runTimes[i], runTicks[i]);
csvWriter.Flush();
using (StreamWriter modelWriter = new StreamWriter(String.Format(@"Benchmark_{0}/DataSet_{1}/Models_{2}/Iter{3}_Run{4}.txt", Name, dataSet.Number, learner.Name().ToLowerInvariant().Replace(' ', '_'), iteration, i)),
modelCSVWriter = new StreamWriter(String.Format(@"Benchmark_{0}/DataSet_{1}/Models_{2}/Iter{3}_Run{4}.csv", Name, dataSet.Number, learner.Name().ToLowerInvariant().Replace(' ', '_'), iteration, i)))
{
modelWriter.AutoFlush = true;
modelCSVWriter.AutoFlush = true;
modelWriter.WriteLine("DataSet {0}", dataSet.Number);
modelWriter.WriteLine("Learner: {0}", learner.Name());
modelWriter.WriteLine("{0}: {1:0000.0000000000}", (useTestData ? "PautomaC Score" : "Log Likelihood"), score);
modelWriter.WriteLine();
learner.Save(modelWriter, modelCSVWriter);
}
}
yield return runScores.Average();
yield return Median(runScores);
yield return runTimes.Average();
yield return Median(runTimes);
}