private TreebankStats.ObservedCorpusStats GatherStats(DiskTreebank tb, string name)
{
TreebankStats.ObservedCorpusStats ocs = new TreebankStats.ObservedCorpusStats(name);
if (makeVocab)
{
trainVocab = Generics.NewHashSet();
}
System.Console.Out.WriteLine("Reading treebank:");
foreach (Tree t in tb)
{
Pair <int, int> treeFacts = DissectTree(t, ocs, makeVocab);
ocs.AddStatsForTree(t.Yield().Count, treeFacts.First(), treeFacts.Second());
if (ocs.numTrees % 100 == 0)
{
System.Console.Out.Write(".");
}
else
{
if (ocs.numTrees % 8001 == 0)
{
System.Console.Out.WriteLine();
}
}
}
ocs.ComputeFinalValues();
System.Console.Out.WriteLine("done!");
return(ocs);
}
