private int typesOfErrors = 2; // shyness and eagerness, can be changed if more different types of errors are found #endregion Fields #region Constructors /// <summary> /// Constructor /// </summary> /// <param name="M_speakers">Number of speakers</param> /// <param name="M_word_types">Number of word types</param> /// <param name="M_sample">Number of samples per word type per speaker</param> /// <param name="d">audio data</param> /// <param name="grammar_directory">output directory of created grammars</param> /// <param name="input">grammar file input</param> /// <param name="number_of_alternates">number of alternative pronunciations</param> /// <param name="r">number of repeats</param> public Tester(int M_speakers, int M_word_types, int M_sample, Data d, string grammar_directory, GrammarReader input, int number_of_alternates, int r) { MWordTypes = M_word_types; MSample = M_sample; MSpeakers = M_speakers; data = d; grammarDirectory = grammar_directory; this.input = input; numberOfAlternates = number_of_alternates; repeats = r; // initialize output SampleAccuracy = new Fraction[MSpeakers - 1, MSample, V.Length, MSpeakers, MWordTypes, MSample]; WordTypeAccuracy = new Fraction[MSpeakers - 1, MSample, V.Length, MSpeakers, MWordTypes]; OverallWordTypeAccuracy = new Fraction[MSpeakers - 1, MSample, V.Length, MWordTypes]; OverallAccuracy = new Fraction[MSpeakers-1, MSample, V.Length]; SampleDeviation = new List<int>[MSpeakers-1,MSample, MSpeakers, MWordTypes]; ConfusionMatrix = new Fraction[MSpeakers - 1, MSpeakers, MWordTypes * MSample, (numberOfAlternates * MWordTypes) + 1]; ErrorRates = new Fraction[MSpeakers - 1, MSpeakers, typesOfErrors, (numberOfAlternates * MWordTypes)]; for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++) { for (int NTrainingSamplesPerSpeakerIndex = 0; NTrainingSamplesPerSpeakerIndex < MSample; NTrainingSamplesPerSpeakerIndex++) { for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++) { OverallAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex] = new Fraction(); for (int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++) { for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++) { WordTypeAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex] = new Fraction(); OverallWordTypeAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iWordTypeIndex] = new Fraction(); SampleDeviation[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, iTestSpeakerIndex, iWordTypeIndex] = new List<int>(); for (int iSamplesIndex = 0; iSamplesIndex < MSample; iSamplesIndex++) { SampleAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex, iSamplesIndex] = new Fraction(); } } } } } } for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++) { for (int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++) { for (int i = 0; i < MSample*MWordTypes; i++) { for (int j = 0; j < (numberOfAlternates*MWordTypes) + 1; j++) { ConfusionMatrix[NTrainingSpeakersIndex, iTestSpeakerIndex, i, j] = new Fraction(); if ( i < typesOfErrors && j < numberOfAlternates*MWordTypes) ErrorRates[NTrainingSpeakersIndex, iTestSpeakerIndex, i, j] = new Fraction(); } } } } }
private string dataOutput(int NTrainingSpeakersIndex, int iTestSpeakerIndex) { string output = ","; // header for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++) { output += V.ElementAt(vocabSizeIndex) + ","; } output += "Overall\n"; // actual data List<string> problemWordTypes = new List<string>(); for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++) { output += data.listOfWords[iWordTypeIndex + 1] + ","; Fraction sum = new Fraction(); for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++) { sum = sum + WordTypeAccuracy[NTrainingSpeakersIndex, 4, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex]; output += WordTypeAccuracy[NTrainingSpeakersIndex, 4, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex] + ","; if (vocabSizeIndex == V.Length - 1 && WordTypeAccuracy[NTrainingSpeakersIndex, 4, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex].toDecimal() < 0.8) problemWordTypes.Add(data.listOfWords[iWordTypeIndex + 1]); } output += sum; output += "\n"; } // overall summary output += "Overall,"; for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++) { Fraction sum = new Fraction(); for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++) { sum = sum + WordTypeAccuracy[NTrainingSpeakersIndex, 4, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex]; } output += sum + ","; } output += "\n"; // problem word types output += "Problem word types,"; foreach (string wordType in problemWordTypes) { output += wordType + ","; } output += "\n"; return output; }
/// <summary> /// Fills in the output data from data in SampeAccuracy /// </summary> public void createOutput() { // fill in the output from SampleAccuracy for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++) { for (int NTrainingSamplesPerSpeakerIndex = 0; NTrainingSamplesPerSpeakerIndex < MSample; NTrainingSamplesPerSpeakerIndex++) { for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++) { Fraction sumOverallAccuracy = new Fraction(); for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++) { Fraction sumOverallWordTypeAccuracy = new Fraction(); for (int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++) { Fraction sumWordTypeAccuracy = new Fraction(); for (int iSamplesIndex = 0; iSamplesIndex < MSample; iSamplesIndex++) { sumWordTypeAccuracy = sumWordTypeAccuracy + SampleAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex, iSamplesIndex]; } WordTypeAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex] = sumWordTypeAccuracy; sumOverallWordTypeAccuracy = sumOverallWordTypeAccuracy + sumWordTypeAccuracy; } OverallWordTypeAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iWordTypeIndex] = sumOverallWordTypeAccuracy; sumOverallAccuracy = sumOverallAccuracy + sumOverallWordTypeAccuracy; } OverallAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex] = sumOverallAccuracy; } } } // fill in SampleDeviation for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++) { for (int NTrainingSamplesPerSpeakerIndex = 0; NTrainingSamplesPerSpeakerIndex < MSample; NTrainingSamplesPerSpeakerIndex++) { for(int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++) { for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++) { for(int iSampleIndex = 0; iSampleIndex < MSample; iSampleIndex++) { // calculate average Fraction average = new Fraction(); Fraction current = new Fraction(); // for every sample except the current one being considered for (int iOtherTrainingSamplesPerSpeakerIndex = 0; iOtherTrainingSamplesPerSpeakerIndex < MSample; iOtherTrainingSamplesPerSpeakerIndex++) { for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++) { if (iOtherTrainingSamplesPerSpeakerIndex == iSampleIndex) { current = current + SampleAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex, iSampleIndex]; } else { average = average + SampleAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex, iOtherTrainingSamplesPerSpeakerIndex]; } } } // end average if (average.toDecimal() - current.toDecimal() > threshold) { SampleDeviation[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, iTestSpeakerIndex, iWordTypeIndex].Add(iSampleIndex); } } } } } } // fill in ErrorRates for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++) { for (int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++) { for (int iCol = 0; iCol < numberOfAlternates * MWordTypes; iCol++) { Fraction eagerness = new Fraction(); Fraction shyness = new Fraction(); int wordTypeIndex = iCol / numberOfAlternates; for (int iRow = 0; iRow < MSample * MWordTypes; iRow++) { if ((iRow / MSample) == wordTypeIndex) { shyness = shyness + ConfusionMatrix[NTrainingSpeakersIndex, iTestSpeakerIndex, iRow, iCol]; } else { eagerness = eagerness + ConfusionMatrix[NTrainingSpeakersIndex, iTestSpeakerIndex, iRow, iCol]; } } ErrorRates[NTrainingSpeakersIndex, iTestSpeakerIndex, 0, iCol] = eagerness; ErrorRates[NTrainingSpeakersIndex, iTestSpeakerIndex, 1, iCol] = shyness; } } } }