private int typesOfErrors = 2; // shyness and eagerness, can be changed if more different types of errors are found
#endregion Fields
#region Constructors
/// <summary>
/// Constructor
/// </summary>
/// <param name="M_speakers">Number of speakers</param>
/// <param name="M_word_types">Number of word types</param>
/// <param name="M_sample">Number of samples per word type per speaker</param>
/// <param name="d">audio data</param>
/// <param name="grammar_directory">output directory of created grammars</param>
/// <param name="input">grammar file input</param>
/// <param name="number_of_alternates">number of alternative pronunciations</param>
/// <param name="r">number of repeats</param>
public Tester(int M_speakers, int M_word_types, int M_sample, Data d, string grammar_directory, GrammarReader input, int number_of_alternates, int r)
{
MWordTypes = M_word_types;
MSample = M_sample;
MSpeakers = M_speakers;
data = d;
grammarDirectory = grammar_directory;
this.input = input;
numberOfAlternates = number_of_alternates;
repeats = r;
// initialize output
SampleAccuracy = new Fraction[MSpeakers - 1, MSample, V.Length, MSpeakers, MWordTypes, MSample];
WordTypeAccuracy = new Fraction[MSpeakers - 1, MSample, V.Length, MSpeakers, MWordTypes];
OverallWordTypeAccuracy = new Fraction[MSpeakers - 1, MSample, V.Length, MWordTypes];
OverallAccuracy = new Fraction[MSpeakers-1, MSample, V.Length];
SampleDeviation = new List<int>[MSpeakers-1,MSample, MSpeakers, MWordTypes];
ConfusionMatrix = new Fraction[MSpeakers - 1, MSpeakers, MWordTypes * MSample, (numberOfAlternates * MWordTypes) + 1];
ErrorRates = new Fraction[MSpeakers - 1, MSpeakers, typesOfErrors, (numberOfAlternates * MWordTypes)];
for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++)
{
for (int NTrainingSamplesPerSpeakerIndex = 0; NTrainingSamplesPerSpeakerIndex < MSample; NTrainingSamplesPerSpeakerIndex++)
{
for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++)
{
OverallAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex] = new Fraction();
for (int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++)
{
for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++)
{
WordTypeAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex] = new Fraction();
OverallWordTypeAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iWordTypeIndex] = new Fraction();
SampleDeviation[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, iTestSpeakerIndex, iWordTypeIndex] = new List<int>();
for (int iSamplesIndex = 0; iSamplesIndex < MSample; iSamplesIndex++)
{
SampleAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex, iSamplesIndex] = new Fraction();
}
}
}
}
}
}
for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++)
{
for (int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++)
{
for (int i = 0; i < MSample*MWordTypes; i++)
{
for (int j = 0; j < (numberOfAlternates*MWordTypes) + 1; j++)
{
ConfusionMatrix[NTrainingSpeakersIndex, iTestSpeakerIndex, i, j] = new Fraction();
if ( i < typesOfErrors && j < numberOfAlternates*MWordTypes)
ErrorRates[NTrainingSpeakersIndex, iTestSpeakerIndex, i, j] = new Fraction();
}
}
}
}
}
private string dataOutput(int NTrainingSpeakersIndex, int iTestSpeakerIndex)
{
string output = ",";
// header
for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++)
{
output += V.ElementAt(vocabSizeIndex) + ",";
}
output += "Overall\n";
// actual data
List<string> problemWordTypes = new List<string>();
for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++)
{
output += data.listOfWords[iWordTypeIndex + 1] + ",";
Fraction sum = new Fraction();
for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++)
{
sum = sum + WordTypeAccuracy[NTrainingSpeakersIndex, 4, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex];
output += WordTypeAccuracy[NTrainingSpeakersIndex, 4, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex] + ",";
if (vocabSizeIndex == V.Length - 1 && WordTypeAccuracy[NTrainingSpeakersIndex, 4, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex].toDecimal() < 0.8)
problemWordTypes.Add(data.listOfWords[iWordTypeIndex + 1]);
}
output += sum;
output += "\n";
}
// overall summary
output += "Overall,";
for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++)
{
Fraction sum = new Fraction();
for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++)
{
sum = sum + WordTypeAccuracy[NTrainingSpeakersIndex, 4, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex];
}
output += sum + ",";
}
output += "\n";
// problem word types
output += "Problem word types,";
foreach (string wordType in problemWordTypes)
{
output += wordType + ",";
}
output += "\n";
return output;
}
/// <summary>
/// Fills in the output data from data in SampeAccuracy
/// </summary>
public void createOutput()
{
// fill in the output from SampleAccuracy
for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++)
{
for (int NTrainingSamplesPerSpeakerIndex = 0; NTrainingSamplesPerSpeakerIndex < MSample; NTrainingSamplesPerSpeakerIndex++)
{
for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++)
{
Fraction sumOverallAccuracy = new Fraction();
for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++)
{
Fraction sumOverallWordTypeAccuracy = new Fraction();
for (int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++)
{
Fraction sumWordTypeAccuracy = new Fraction();
for (int iSamplesIndex = 0; iSamplesIndex < MSample; iSamplesIndex++)
{
sumWordTypeAccuracy = sumWordTypeAccuracy + SampleAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex, iSamplesIndex];
}
WordTypeAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex] = sumWordTypeAccuracy;
sumOverallWordTypeAccuracy = sumOverallWordTypeAccuracy + sumWordTypeAccuracy;
}
OverallWordTypeAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iWordTypeIndex] = sumOverallWordTypeAccuracy;
sumOverallAccuracy = sumOverallAccuracy + sumOverallWordTypeAccuracy;
}
OverallAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex] = sumOverallAccuracy;
}
}
}
// fill in SampleDeviation
for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++)
{
for (int NTrainingSamplesPerSpeakerIndex = 0; NTrainingSamplesPerSpeakerIndex < MSample; NTrainingSamplesPerSpeakerIndex++)
{
for(int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++)
{
for (int iWordTypeIndex = 0; iWordTypeIndex < MWordTypes; iWordTypeIndex++)
{
for(int iSampleIndex = 0; iSampleIndex < MSample; iSampleIndex++)
{
// calculate average
Fraction average = new Fraction();
Fraction current = new Fraction();
// for every sample except the current one being considered
for (int iOtherTrainingSamplesPerSpeakerIndex = 0; iOtherTrainingSamplesPerSpeakerIndex < MSample; iOtherTrainingSamplesPerSpeakerIndex++)
{
for (int vocabSizeIndex = 0; vocabSizeIndex < V.Length; vocabSizeIndex++)
{
if (iOtherTrainingSamplesPerSpeakerIndex == iSampleIndex)
{
current = current + SampleAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex, iSampleIndex];
}
else
{
average = average + SampleAccuracy[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, vocabSizeIndex, iTestSpeakerIndex, iWordTypeIndex, iOtherTrainingSamplesPerSpeakerIndex];
}
}
}
// end average
if (average.toDecimal() - current.toDecimal() > threshold)
{
SampleDeviation[NTrainingSpeakersIndex, NTrainingSamplesPerSpeakerIndex, iTestSpeakerIndex, iWordTypeIndex].Add(iSampleIndex);
}
}
}
}
}
}
// fill in ErrorRates
for (int NTrainingSpeakersIndex = 0; NTrainingSpeakersIndex < MSpeakers - 1; NTrainingSpeakersIndex++)
{
for (int iTestSpeakerIndex = 0; iTestSpeakerIndex < MSpeakers; iTestSpeakerIndex++)
{
for (int iCol = 0; iCol < numberOfAlternates * MWordTypes; iCol++)
{
Fraction eagerness = new Fraction();
Fraction shyness = new Fraction();
int wordTypeIndex = iCol / numberOfAlternates;
for (int iRow = 0; iRow < MSample * MWordTypes; iRow++)
{
if ((iRow / MSample) == wordTypeIndex)
{
shyness = shyness + ConfusionMatrix[NTrainingSpeakersIndex, iTestSpeakerIndex, iRow, iCol];
}
else
{
eagerness = eagerness + ConfusionMatrix[NTrainingSpeakersIndex, iTestSpeakerIndex, iRow, iCol];
}
}
ErrorRates[NTrainingSpeakersIndex, iTestSpeakerIndex, 0, iCol] = eagerness;
ErrorRates[NTrainingSpeakersIndex, iTestSpeakerIndex, 1, iCol] = shyness;
}
}
}
}
