public virtual void TestValueSort()
{
string expected = " Guess/Gold 2 1 Marg. (Guess)\n" + " 2 3 2 5\n" + " 1 5 10 15\n" + " Marg. (Gold) 8 12\n\n" + " 2 prec=0.6, recall=0.375, spec=0.83333, f1=0.46154\n"
+ " 1 prec=0.66667, recall=0.83333, spec=0.375, f1=0.74074\n";
ConfusionMatrixTest.BackwardsInteger one = new ConfusionMatrixTest.BackwardsInteger(1);
ConfusionMatrixTest.BackwardsInteger two = new ConfusionMatrixTest.BackwardsInteger(2);
ConfusionMatrix <ConfusionMatrixTest.BackwardsInteger> conf = new ConfusionMatrix <ConfusionMatrixTest.BackwardsInteger>(Locale.Us);
conf.SetUseRealLabels(true);
conf.Add(one, one, 10);
conf.Add(one, two, 5);
conf.Add(two, one, 2);
conf.Add(two, two, 3);
string result = conf.PrintTable();
if (echo)
{
System.Console.Error.WriteLine(result);
}
else
{
NUnit.Framework.Assert.AreEqual(expected, result);
}
}
public virtual void Predict(ICollection<string> predictedRelationsRaw, ICollection<string> goldRelationsRaw)
{
ICollection<string> predictedRelations = new HashSet<string>(predictedRelationsRaw);
predictedRelations.Remove(KBPRelationExtractorConstants.NoRelation);
ICollection<string> goldRelations = new HashSet<string>(goldRelationsRaw);
goldRelations.Remove(KBPRelationExtractorConstants.NoRelation);
// Register the prediction
foreach (string pred in predictedRelations)
{
if (goldRelations.Contains(pred))
{
correctCount.IncrementCount(pred);
}
predictedCount.IncrementCount(pred);
}
goldRelations.ForEach(null);
HashSet<string> allRelations = new HashSet<string>();
Sharpen.Collections.AddAll(allRelations, predictedRelations);
Sharpen.Collections.AddAll(allRelations, goldRelations);
allRelations.ForEach(null);
// Register the confusion matrix
if (predictedRelations.Count == 1 && goldRelations.Count == 1)
{
confusion.Add(predictedRelations.GetEnumerator().Current, goldRelations.GetEnumerator().Current);
}
if (predictedRelations.Count == 1 && goldRelations.IsEmpty())
{
confusion.Add(predictedRelations.GetEnumerator().Current, "NR");
}
if (predictedRelations.IsEmpty() && goldRelations.Count == 1)
{
confusion.Add("NR", goldRelations.GetEnumerator().Current);
}
}
/// <summary>Update a confusion matrix with the errors from this sentence.</summary>
/// <param name="finalTags">Chosen tags for sentence</param>
/// <param name="confusionMatrix">Confusion matrix to write to</param>
protected internal virtual void UpdateConfusionMatrix(string[] finalTags, ConfusionMatrix <string> confusionMatrix)
{
for (int i = 0; i < correctTags.Length; i++)
{
confusionMatrix.Add(finalTags[i], correctTags[i]);
}
}
public static ConfusionMatrix CrossValidation(DataTable data, int folds)
{
DataTable t = CollectionExtensions.OrderRandomly(data.AsEnumerable()).CopyToDataTable();
ConfusionMatrix res = new ConfusionMatrix(classValues);
int n = t.Rows.Count;
int c = (int)Math.Floor(n / (double)folds);
int start = 0;
int stop = c - 1;
int fold = 1;
while (start < n) {
if (stop > n) { stop = n; }
//use start and stop to get a subset of data as test
//set everything else as train
IEnumerable<DataRow> testEnum = null, trainEnum = null;
CollectionExtensions.Split(t.AsEnumerable(), start, stop, ref testEnum, ref trainEnum);
DataTable trainingData = trainEnum.CopyToDataTable();
start = stop + 1;
stop = start - 1 + c;
Train(trainingData);
ConfusionMatrix subres = Test(testEnum.CopyToDataTable());
res.Add(subres); //also updates the accuracy
fold++;
}
return res;
}
public virtual void TestBulkAdd()
{
string expected = " Guess/Gold C1 C2 Marg. (Guess)\n" + " C1 10 5 15\n" + " C2 2 3 5\n" + " Marg. (Gold) 12 8\n\n" + " C1 = 1 prec=0.66667, recall=0.83333, spec=0.375, f1=0.74074\n"
+ " C2 = 2 prec=0.6, recall=0.375, spec=0.83333, f1=0.46154\n";
ConfusionMatrix <int> conf = new ConfusionMatrix <int>(Locale.Us);
conf.Add(1, 1, 10);
conf.Add(1, 2, 5);
conf.Add(2, 1, 2);
conf.Add(2, 2, 3);
string result = conf.PrintTable();
if (echo)
{
System.Console.Error.WriteLine(result);
}
else
{
NUnit.Framework.Assert.AreEqual(expected, result);
}
}
public virtual void TestBasic()
{
string expected = " Guess/Gold C1 C2 C3 Marg. (Guess)\n" + " C1 2 0 0 2\n" + " C2 1 0 0 1\n" + " C3 0 0 1 1\n"
+ " Marg. (Gold) 3 0 1\n\n" + " C1 = a prec=1, recall=0.66667, spec=1, f1=0.8\n" + " C2 = b prec=0, recall=n/a, spec=0.75, f1=n/a\n" + " C3 = c prec=1, recall=1, spec=1, f1=1\n";
ConfusionMatrix <string> conf = new ConfusionMatrix <string>(Locale.Us);
conf.Add("a", "a");
conf.Add("a", "a");
conf.Add("b", "a");
conf.Add("c", "c");
string result = conf.PrintTable();
if (echo)
{
System.Console.Error.WriteLine(result);
}
else
{
NUnit.Framework.Assert.AreEqual(expected, result);
}
}
public virtual void TestRealLabels()
{
string expected = " Guess/Gold a b c Marg. (Guess)\n" + " a 2 0 0 2\n" + " b 1 0 0 1\n" + " c 0 0 1 1\n"
+ " Marg. (Gold) 3 0 1\n\n" + " a prec=1, recall=0.66667, spec=1, f1=0.8\n" + " b prec=0, recall=n/a, spec=0.75, f1=n/a\n" + " c prec=1, recall=1, spec=1, f1=1\n";
ConfusionMatrix <string> conf = new ConfusionMatrix <string>(Locale.Us);
conf.SetUseRealLabels(true);
conf.Add("a", "a");
conf.Add("a", "a");
conf.Add("b", "a");
conf.Add("c", "c");
string result = conf.PrintTable();
if (echo)
{
System.Console.Error.WriteLine(result);
}
else
{
NUnit.Framework.Assert.AreEqual(expected, result);
}
}
protected internal static void PrintConfusionMatrix(string name, int[][] confusion)
{
log.Info(name + " confusion matrix");
ConfusionMatrix <int> confusionMatrix = new ConfusionMatrix <int>();
confusionMatrix.SetUseRealLabels(true);
for (int i = 0; i < confusion.Length; ++i)
{
for (int j = 0; j < confusion[i].Length; ++j)
{
confusionMatrix.Add(j, i, confusion[i][j]);
}
}
log.Info(confusionMatrix);
}
public static void Main(string[] args)
{
if (args.Length < 1)
{
System.Console.Error.Printf("Usage: java %s answers_file%n", typeof(ConfusionMatrix).FullName);
System.Environment.Exit(-1);
}
try
{
ConfusionMatrix <string> cm = new ConfusionMatrix <string>();
string answersFile = args[0];
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(answersFile), "UTF-8"));
string line = br.ReadLine();
for (; line != null; line = br.ReadLine())
{
string[] tokens = line.Split("\\s");
if (tokens.Length != 3)
{
System.Console.Error.Printf("ignoring bad line");
continue;
}
//System.exit(-1);
cm.Add(tokens[2], tokens[1]);
}
System.Console.Out.WriteLine(cm.ToString());
}
catch (UnsupportedEncodingException e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
catch (FileNotFoundException e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
catch (IOException e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
static void Main(string[] args)
{
if (args.Length == 0)
{
return;
}
List <string> allLabels = new List <string>(leafLabels);
allLabels.Add("Wire");
Dictionary <string, List <string>[]> user2sketches = GetSketchesPerUser(args[0], args[1]);
Dictionary <string, List <Substroke> > user2gateStrokes = new Dictionary <string, List <Substroke> >();
Dictionary <string, List <Substroke> > user2otherStrokes = new Dictionary <string, List <Substroke> >();
foreach (KeyValuePair <string, List <string>[]> pair in user2sketches)
{
string user = pair.Key;
user2gateStrokes.Add(user, new List <Substroke>());
user2otherStrokes.Add(user, new List <Substroke>());
int userNum;
bool good = int.TryParse(user.Substring(0, 2), out userNum);
if (good && userNum <= 0)
{
continue;
}
List <string> sketchFiles = pair.Value[0];
List <string> testFiles = pair.Value[1];
foreach (string sketchFile in sketchFiles)
{
Sketch.Sketch sketch = new ReadXML(sketchFile).Sketch;
sketch = General.ReOrderParentShapes(sketch);
foreach (Substroke s in sketch.Substrokes)
{
if (s.Labels.Length > 0)
{
string label = s.Labels[s.Labels.Length - 1];
if (leafLabels.Contains(label))
{
user2gateStrokes[user].Add(s);
}
}
}
}
foreach (string sketchFile in testFiles)
{
Sketch.Sketch sketch = new ReadXML(sketchFile).Sketch;
sketch = General.ReOrderParentShapes(sketch);
foreach (Substroke s in sketch.Substrokes)
{
if (s.Labels.Length > 0)
{
string label = s.Labels[s.Labels.Length - 1];
string root = s.Labels[0];
if (label == "Wire" || label == "LabelBoxOther")// || root == "LabelBox")
{
user2otherStrokes[user].Add(s);
}
}
}
}
}
Dictionary <string, ConfusionMatrix> user2Confusion = new Dictionary <string, ConfusionMatrix>();
ConfusionMatrix allConfusion = new ConfusionMatrix(allLabels);
// Use this to train a rubine recognizer on all substrokes...
//RubineRecognizerUpdateable rubine = new RubineRecognizerUpdateable();
foreach (string user in user2gateStrokes.Keys)
{
RubineRecognizerUpdateable rubine = new RubineRecognizerUpdateable();
foreach (Substroke s in user2gateStrokes[user])
{
if (s.Labels.Length > 0)
{
rubine.Add(s.Labels[s.Labels.Length - 1], s.PointsL);
}
}
foreach (KeyValuePair <string, List <Substroke> > kv in user2otherStrokes)
{
if (kv.Key.Substring(0, 2) != user.Substring(0, 2))
{
foreach (Substroke s in kv.Value)
{
if (s.Labels.Length > 0)
{
} //rubine.Add(s.Labels[s.Labels.Length - 1], s.PointsL);
}
}
}
bool updated = rubine.updateMatrices();
if (!updated)
{
continue;
}
ConfusionMatrix confusion = new ConfusionMatrix(allLabels);
foreach (string sketchFile in user2sketches[user][1])
{
Sketch.Sketch sketch = new ReadXML(sketchFile).Sketch;
sketch = General.ReOrderParentShapes(sketch);
foreach (Substroke s in sketch.Substrokes)
{
if (s.Labels.Length > 0)
{
string label = s.Labels[s.Labels.Length - 1];
if (leafLabels.Contains(label))
{
string result = rubine.Recognize(s);
confusion.Add(label, result);
allConfusion.Add(label, result);
}
}
}
}
confusion.IncreaseAllByOne();
user2Confusion.Add(user, confusion);
}
allConfusion.IncreaseAllByOne();
System.IO.StreamWriter allWriter = new System.IO.StreamWriter("allConfusion.txt");
allConfusion.Print(allWriter);
allWriter.Close();
foreach (KeyValuePair <string, ConfusionMatrix> kv in user2Confusion)
{
string user = kv.Key;
System.IO.StreamWriter writer = new System.IO.StreamWriter("Confusion" + user + ".txt");
kv.Value.Print(writer);
writer.Close();
}
}