public static double Evaluate(Learner learner, SequenceData testSequences, double[] solutionData) {
     double[] guessedProbs = new double[testSequences.Count];
     for (int i = 0; i < guessedProbs.Length; i++)
         guessedProbs[i] = learner.CalculateProbability(testSequences[i]);
     guessedProbs = Normalize(guessedProbs);
     return Evaluate(guessedProbs, solutionData);
 }
        public IEnumerable<double> RunLearner(DataSet dataSet, Learner learner, StreamWriter outputWriter, StreamWriter csvWriter, int iteration)
        {
            double[] runScores = new double[numberOfRuns];
            double[] runTimes = new double[numberOfRuns];
            double[] runTicks = new double[numberOfRuns];

            Stopwatch watch = new Stopwatch();

            for (int i = 0; i < numberOfRuns; i++)
            {
                Console.WriteLine("Run {0}...", (i + 1));

                watch.Reset();

                dataSet.SplitData(2.0 / 3.0, i);

                watch.Start();

                learner.Learn(dataSet.TrainingData, dataSet.ValidationData, dataSet.TestData);

                watch.Stop();

                double score = 0.0;

                if (useTestData)
                {
                    score = PautomacEvaluator.Evaluate(learner, dataSet.TestData, dataSet.SolutionData);
                }
                else
                {
                    foreach (int[] signal in dataSet.ValidationData.GetAll())
                    {
                        score -= learner.CalculateProbability(signal, true);
                    }
                }

                runScores[i] = score;
                runTimes[i] = (watch.ElapsedMilliseconds / 1000.0);
                runTicks[i] = watch.ElapsedTicks;

                outputWriter.WriteLine("Run {0:00}:\t{1:00000000.0000000000}\t{2:000000}\t{3:0000000000000000}", (i + 1), runScores[i], runTimes[i], runTicks[i]);
                outputWriter.Flush();

                csvWriter.WriteLine("{0},{1},{2},{3},{4},", iteration, i, runScores[i], runTimes[i], runTicks[i]);
                csvWriter.Flush();
                
                using (StreamWriter modelWriter = new StreamWriter(String.Format(@"Benchmark_{0}/DataSet_{1}/Models_{2}/Iter{3}_Run{4}.txt", Name, dataSet.Number, learner.Name().ToLowerInvariant().Replace(' ', '_'), iteration, i)),
                                    modelCSVWriter = new StreamWriter(String.Format(@"Benchmark_{0}/DataSet_{1}/Models_{2}/Iter{3}_Run{4}.csv", Name, dataSet.Number, learner.Name().ToLowerInvariant().Replace(' ', '_'), iteration, i)))
                {
                    modelWriter.AutoFlush = true;
                    modelCSVWriter.AutoFlush = true;

                    modelWriter.WriteLine("DataSet {0}", dataSet.Number);
                    modelWriter.WriteLine("Learner: {0}", learner.Name());
                    modelWriter.WriteLine("{0}: {1:0000.0000000000}", (useTestData ? "PautomaC Score" : "Log Likelihood"), score);

                    modelWriter.WriteLine();

                    learner.Save(modelWriter, modelCSVWriter);
                }
            }

            yield return runScores.Average();
            yield return Median(runScores);
            yield return runTimes.Average();
            yield return Median(runTimes);
        }