public virtual void InitFiles(ITreebankLangParserParams tlpParams, string goldFilename, string testFilename)
{
try
{
goldWriter = tlpParams.Pw(new FileOutputStream(goldFilename));
testWriter = tlpParams.Pw(new FileOutputStream(testFilename));
}
catch (IOException e)
{
throw new Exception(e);
}
count = 0;
}
public static void Main(string[] args)
{
if (args.Length < minArgs)
{
System.Console.Out.WriteLine(Usage());
System.Environment.Exit(-1);
}
Properties options = StringUtils.ArgsToProperties(args, ArgDefs());
Language language = PropertiesUtils.Get(options, "l", Language.English, typeof(Language));
ITreebankLangParserParams tlpp = language.@params;
DiskTreebank tb = null;
string encoding = options.GetProperty("l", "UTF-8");
bool removeBracket = PropertiesUtils.GetBool(options, "b", false);
tlpp.SetInputEncoding(encoding);
tlpp.SetOutputEncoding(encoding);
tb = tlpp.DiskTreebank();
string[] files = options.GetProperty(string.Empty, string.Empty).Split("\\s+");
if (files.Length != 0)
{
foreach (string filename in files)
{
tb.LoadPath(filename);
}
}
else
{
log.Info(Usage());
System.Environment.Exit(-1);
}
PrintWriter pwo = tlpp.Pw();
string startSymbol = tlpp.TreebankLanguagePack().StartSymbol();
ITreeFactory tf = new LabeledScoredTreeFactory();
int nTrees = 0;
foreach (Tree t in tb)
{
if (removeBracket)
{
if (t.Value().Equals(startSymbol))
{
t = t.FirstChild();
}
}
else
{
if (!t.Value().Equals(startSymbol))
{
//Add a bracket if it isn't already there
t = tf.NewTreeNode(startSymbol, Java.Util.Collections.SingletonList(t));
}
}
pwo.Println(t.ToString());
nTrees++;
}
pwo.Close();
System.Console.Error.Printf("Processed %d trees.%n", nTrees);
}
/// <summary>
/// This method lets you train and test a segmenter relative to a
/// Treebank.
/// </summary>
/// <remarks>
/// This method lets you train and test a segmenter relative to a
/// Treebank.
/// <p>
/// <i>Implementation note:</i> This method is largely cloned from
/// LexicalizedParser's main method. Should we try to have it be able
/// to train segmenters to stop things going out of sync?
/// </remarks>
public static void Main(string[] args)
{
bool train = false;
bool saveToSerializedFile = false;
bool saveToTextFile = false;
string serializedInputFileOrUrl = null;
string textInputFileOrUrl = null;
string serializedOutputFileOrUrl = null;
string textOutputFileOrUrl = null;
string treebankPath = null;
Treebank testTreebank = null;
// Treebank tuneTreebank = null;
string testPath = null;
IFileFilter testFilter = null;
IFileFilter trainFilter = null;
string encoding = null;
// variables needed to process the files to be parsed
ITokenizerFactory <Word> tokenizerFactory = null;
// DocumentPreprocessor documentPreprocessor = new DocumentPreprocessor();
bool tokenized = false;
// whether or not the input file has already been tokenized
IFunction <IList <IHasWord>, IList <IHasWord> > escaper = new ChineseEscaper();
// int tagDelimiter = -1;
// String sentenceDelimiter = "\n";
// boolean fromXML = false;
int argIndex = 0;
if (args.Length < 1)
{
log.Info("usage: java edu.stanford.nlp.parser.lexparser." + "LexicalizedParser parserFileOrUrl filename*");
return;
}
Options op = new Options();
op.tlpParams = new ChineseTreebankParserParams();
// while loop through option arguments
while (argIndex < args.Length && args[argIndex][0] == '-')
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-train"))
{
train = true;
saveToSerializedFile = true;
int numSubArgs = NumSubArgs(args, argIndex);
argIndex++;
if (numSubArgs > 1)
{
treebankPath = args[argIndex];
argIndex++;
}
else
{
throw new Exception("Error: -train option must have treebankPath as first argument.");
}
if (numSubArgs == 2)
{
trainFilter = new NumberRangesFileFilter(args[argIndex++], true);
}
else
{
if (numSubArgs >= 3)
{
try
{
int low = System.Convert.ToInt32(args[argIndex]);
int high = System.Convert.ToInt32(args[argIndex + 1]);
trainFilter = new NumberRangeFileFilter(low, high, true);
argIndex += 2;
}
catch (NumberFormatException)
{
// maybe it's a ranges expression?
trainFilter = new NumberRangesFileFilter(args[argIndex], true);
argIndex++;
}
}
}
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-encoding"))
{
// sets encoding for TreebankLangParserParams
encoding = args[argIndex + 1];
op.tlpParams.SetInputEncoding(encoding);
op.tlpParams.SetOutputEncoding(encoding);
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-loadFromSerializedFile"))
{
// load the parser from a binary serialized file
// the next argument must be the path to the parser file
serializedInputFileOrUrl = args[argIndex + 1];
argIndex += 2;
}
else
{
// doesn't make sense to load from TextFile -pichuan
// } else if (args[argIndex].equalsIgnoreCase("-loadFromTextFile")) {
// // load the parser from declarative text file
// // the next argument must be the path to the parser file
// textInputFileOrUrl = args[argIndex + 1];
// argIndex += 2;
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-saveToSerializedFile"))
{
saveToSerializedFile = true;
serializedOutputFileOrUrl = args[argIndex + 1];
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-saveToTextFile"))
{
// save the parser to declarative text file
saveToTextFile = true;
textOutputFileOrUrl = args[argIndex + 1];
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-treebank"))
{
// the next argument is the treebank path and range for testing
int numSubArgs = NumSubArgs(args, argIndex);
argIndex++;
if (numSubArgs == 1)
{
testFilter = new NumberRangesFileFilter(args[argIndex++], true);
}
else
{
if (numSubArgs > 1)
{
testPath = args[argIndex++];
if (numSubArgs == 2)
{
testFilter = new NumberRangesFileFilter(args[argIndex++], true);
}
else
{
if (numSubArgs >= 3)
{
try
{
int low = System.Convert.ToInt32(args[argIndex]);
int high = System.Convert.ToInt32(args[argIndex + 1]);
testFilter = new NumberRangeFileFilter(low, high, true);
argIndex += 2;
}
catch (NumberFormatException)
{
// maybe it's a ranges expression?
testFilter = new NumberRangesFileFilter(args[argIndex++], true);
}
}
}
}
}
}
else
{
int j = op.tlpParams.SetOptionFlag(args, argIndex);
if (j == argIndex)
{
log.Info("Unknown option ignored: " + args[argIndex]);
j++;
}
argIndex = j;
}
}
}
}
}
}
}
// end while loop through arguments
ITreebankLangParserParams tlpParams = op.tlpParams;
// all other arguments are order dependent and
// are processed in order below
Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter cs = null;
if (!train && op.testOptions.verbose)
{
System.Console.Out.WriteLine("Currently " + new DateTime());
PrintArgs(args, System.Console.Out);
}
if (train)
{
PrintArgs(args, System.Console.Out);
// so we train a parser using the treebank
if (treebankPath == null)
{
// the next arg must be the treebank path, since it wasn't give earlier
treebankPath = args[argIndex];
argIndex++;
if (args.Length > argIndex + 1)
{
try
{
// the next two args might be the range
int low = System.Convert.ToInt32(args[argIndex]);
int high = System.Convert.ToInt32(args[argIndex + 1]);
trainFilter = new NumberRangeFileFilter(low, high, true);
argIndex += 2;
}
catch (NumberFormatException)
{
// maybe it's a ranges expression?
trainFilter = new NumberRangesFileFilter(args[argIndex], true);
argIndex++;
}
}
}
Treebank trainTreebank = MakeTreebank(treebankPath, op, trainFilter);
IIndex <string> wordIndex = new HashIndex <string>();
IIndex <string> tagIndex = new HashIndex <string>();
cs = new Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter(trainTreebank, op, wordIndex, tagIndex);
}
else
{
if (textInputFileOrUrl != null)
{
}
else
{
// so we load the segmenter from a text grammar file
// XXXXX fix later -pichuan
//cs = new LexicalizedParser(textInputFileOrUrl, true, op);
// so we load a serialized segmenter
if (serializedInputFileOrUrl == null)
{
// the next argument must be the path to the serialized parser
serializedInputFileOrUrl = args[argIndex];
argIndex++;
}
try
{
cs = new Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter(serializedInputFileOrUrl, op);
}
catch (ArgumentException)
{
log.Info("Error loading segmenter, exiting...");
System.Environment.Exit(0);
}
}
}
// the following has to go after reading parser to make sure
// op and tlpParams are the same for train and test
TreePrint treePrint = op.testOptions.TreePrint(tlpParams);
if (testFilter != null)
{
if (testPath == null)
{
if (treebankPath == null)
{
throw new Exception("No test treebank path specified...");
}
else
{
log.Info("No test treebank path specified. Using train path: \"" + treebankPath + "\"");
testPath = treebankPath;
}
}
testTreebank = tlpParams.TestMemoryTreebank();
testTreebank.LoadPath(testPath, testFilter);
}
op.trainOptions.sisterSplitters = Generics.NewHashSet(Arrays.AsList(tlpParams.SisterSplitters()));
// at this point we should be sure that op.tlpParams is
// set appropriately (from command line, or from grammar file),
// and will never change again. We also set the tlpParams of the
// LexicalizedParser instance to be the same object. This is
// redundancy that we probably should take out eventually.
//
// -- Roger
if (op.testOptions.verbose)
{
log.Info("Lexicon is " + cs.GetType().FullName);
}
PrintWriter pwOut = tlpParams.Pw();
PrintWriter pwErr = tlpParams.Pw(System.Console.Error);
// Now what do we do with the parser we've made
if (saveToTextFile)
{
// save the parser to textGrammar format
if (textOutputFileOrUrl != null)
{
SaveSegmenterDataToText(cs, textOutputFileOrUrl);
}
else
{
log.Info("Usage: must specify a text segmenter data output path");
}
}
if (saveToSerializedFile)
{
if (serializedOutputFileOrUrl == null && argIndex < args.Length)
{
// the next argument must be the path to serialize to
serializedOutputFileOrUrl = args[argIndex];
argIndex++;
}
if (serializedOutputFileOrUrl != null)
{
SaveSegmenterDataToSerialized(cs, serializedOutputFileOrUrl);
}
else
{
if (textOutputFileOrUrl == null && testTreebank == null)
{
// no saving/parsing request has been specified
log.Info("usage: " + "java edu.stanford.nlp.parser.lexparser.ChineseLexiconAndWordSegmenter" + "-train trainFilesPath [start stop] serializedParserFilename");
}
}
}
/* --------------------- Testing part!!!! ----------------------- */
if (op.testOptions.verbose)
{
}
// printOptions(false, op);
if (testTreebank != null || (argIndex < args.Length && Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-treebank")))
{
// test parser on treebank
if (testTreebank == null)
{
// the next argument is the treebank path and range for testing
testTreebank = tlpParams.TestMemoryTreebank();
if (args.Length < argIndex + 4)
{
testTreebank.LoadPath(args[argIndex + 1]);
}
else
{
int testlow = System.Convert.ToInt32(args[argIndex + 2]);
int testhigh = System.Convert.ToInt32(args[argIndex + 3]);
testTreebank.LoadPath(args[argIndex + 1], new NumberRangeFileFilter(testlow, testhigh, true));
}
}
}
}
/// <summary>Run the Evalb scoring metric on guess/gold input.</summary>
/// <remarks>Run the Evalb scoring metric on guess/gold input. The default language is English.</remarks>
/// <param name="args"/>
public static void Main(string[] args)
{
if (args.Length < minArgs)
{
log.Info(Usage());
System.Environment.Exit(-1);
}
Properties options = StringUtils.ArgsToProperties(args, OptionArgDefs());
Language language = PropertiesUtils.Get(options, "l", Language.English, typeof(Language));
ITreebankLangParserParams tlpp = language.@params;
int maxGoldYield = PropertiesUtils.GetInt(options, "y", int.MaxValue);
bool Verbose = PropertiesUtils.GetBool(options, "v", false);
bool sortByF1 = PropertiesUtils.HasProperty(options, "s");
int worstKTreesToEmit = PropertiesUtils.GetInt(options, "s", 0);
PriorityQueue <Triple <double, Tree, Tree> > queue = sortByF1 ? new PriorityQueue <Triple <double, Tree, Tree> >(2000, new Evalb.F1Comparator()) : null;
bool doCatLevel = PropertiesUtils.GetBool(options, "c", false);
string labelRegex = options.GetProperty("f", null);
string encoding = options.GetProperty("e", "UTF-8");
string[] parsedArgs = options.GetProperty(string.Empty, string.Empty).Split("\\s+");
if (parsedArgs.Length != minArgs)
{
log.Info(Usage());
System.Environment.Exit(-1);
}
string goldFile = parsedArgs[0];
string guessFile = parsedArgs[1];
// Command-line has been parsed. Configure the metric for evaluation.
tlpp.SetInputEncoding(encoding);
PrintWriter pwOut = tlpp.Pw();
Treebank guessTreebank = tlpp.DiskTreebank();
guessTreebank.LoadPath(guessFile);
pwOut.Println("GUESS TREEBANK:");
pwOut.Println(guessTreebank.TextualSummary());
Treebank goldTreebank = tlpp.DiskTreebank();
goldTreebank.LoadPath(goldFile);
pwOut.Println("GOLD TREEBANK:");
pwOut.Println(goldTreebank.TextualSummary());
Evalb metric = new Evalb("Evalb LP/LR", true);
EvalbByCat evalbCat = (doCatLevel) ? new EvalbByCat("EvalbByCat LP/LR", true, labelRegex) : null;
ITreeTransformer tc = tlpp.Collinizer();
//The evalb ref implementation assigns status for each tree pair as follows:
//
// 0 - Ok (yields match)
// 1 - length mismatch
// 2 - null parse e.g. (()).
//
//In the cases of 1,2, evalb does not include the tree pair in the LP/LR computation.
IEnumerator <Tree> goldItr = goldTreebank.GetEnumerator();
IEnumerator <Tree> guessItr = guessTreebank.GetEnumerator();
int goldLineId = 0;
int guessLineId = 0;
int skippedGuessTrees = 0;
while (guessItr.MoveNext() && goldItr.MoveNext())
{
Tree guessTree = guessItr.Current;
IList <ILabel> guessYield = guessTree.Yield();
guessLineId++;
Tree goldTree = goldItr.Current;
IList <ILabel> goldYield = goldTree.Yield();
goldLineId++;
// Check that we should evaluate this tree
if (goldYield.Count > maxGoldYield)
{
skippedGuessTrees++;
continue;
}
// Only trees with equal yields can be evaluated
if (goldYield.Count != guessYield.Count)
{
pwOut.Printf("Yield mismatch gold: %d tokens vs. guess: %d tokens (lines: gold %d guess %d)%n", goldYield.Count, guessYield.Count, goldLineId, guessLineId);
skippedGuessTrees++;
continue;
}
Tree evalGuess = tc.TransformTree(guessTree);
Tree evalGold = tc.TransformTree(goldTree);
metric.Evaluate(evalGuess, evalGold, ((Verbose) ? pwOut : null));
if (doCatLevel)
{
evalbCat.Evaluate(evalGuess, evalGold, ((Verbose) ? pwOut : null));
}
if (sortByF1)
{
StoreTrees(queue, guessTree, goldTree, metric.GetLastF1());
}
}
if (guessItr.MoveNext() || goldItr.MoveNext())
{
System.Console.Error.Printf("Guess/gold files do not have equal lengths (guess: %d gold: %d)%n.", guessLineId, goldLineId);
}
pwOut.Println("================================================================================");
if (skippedGuessTrees != 0)
{
pwOut.Printf("%s %d guess trees\n", "Unable to evaluate", skippedGuessTrees);
}
metric.Display(true, pwOut);
pwOut.Println();
if (doCatLevel)
{
evalbCat.Display(true, pwOut);
pwOut.Println();
}
if (sortByF1)
{
EmitSortedTrees(queue, worstKTreesToEmit, guessFile);
}
pwOut.Close();
}
/// <returns>A Triple of binaryTrainTreebank, binarySecondaryTreebank, binaryTuneTreebank.</returns>
public static Triple <Treebank, Treebank, Treebank> GetAnnotatedBinaryTreebankFromTreebank(Treebank trainTreebank, Treebank secondaryTreebank, Treebank tuneTreebank, Options op)
{
// setup tree transforms
ITreebankLangParserParams tlpParams = op.tlpParams;
ITreebankLanguagePack tlp = tlpParams.TreebankLanguagePack();
if (op.testOptions.verbose)
{
PrintWriter pwErr = tlpParams.Pw(System.Console.Error);
pwErr.Print("Training ");
pwErr.Println(trainTreebank.TextualSummary(tlp));
if (secondaryTreebank != null)
{
pwErr.Print("Secondary training ");
pwErr.Println(secondaryTreebank.TextualSummary(tlp));
}
}
CompositeTreeTransformer trainTransformer = new CompositeTreeTransformer();
if (op.trainOptions.preTransformer != null)
{
trainTransformer.AddTransformer(op.trainOptions.preTransformer);
}
if (op.trainOptions.collinsPunc)
{
CollinsPuncTransformer collinsPuncTransformer = new CollinsPuncTransformer(tlp);
trainTransformer.AddTransformer(collinsPuncTransformer);
}
log.Info("Binarizing trees...");
Edu.Stanford.Nlp.Parser.Lexparser.TreeAnnotatorAndBinarizer binarizer;
if (!op.trainOptions.leftToRight)
{
binarizer = new Edu.Stanford.Nlp.Parser.Lexparser.TreeAnnotatorAndBinarizer(tlpParams, op.forceCNF, !op.trainOptions.OutsideFactor(), !op.trainOptions.predictSplits, op);
}
else
{
binarizer = new Edu.Stanford.Nlp.Parser.Lexparser.TreeAnnotatorAndBinarizer(tlpParams.HeadFinder(), new LeftHeadFinder(), tlpParams, op.forceCNF, !op.trainOptions.OutsideFactor(), !op.trainOptions.predictSplits, op);
}
trainTransformer.AddTransformer(binarizer);
if (op.wordFunction != null)
{
ITreeTransformer wordFunctionTransformer = new TreeLeafLabelTransformer(op.wordFunction);
trainTransformer.AddTransformer(wordFunctionTransformer);
}
Treebank wholeTreebank;
if (secondaryTreebank == null)
{
wholeTreebank = trainTreebank;
}
else
{
wholeTreebank = new CompositeTreebank(trainTreebank, secondaryTreebank);
}
if (op.trainOptions.selectiveSplit)
{
op.trainOptions.splitters = ParentAnnotationStats.GetSplitCategories(wholeTreebank, op.trainOptions.tagSelectiveSplit, 0, op.trainOptions.selectiveSplitCutOff, op.trainOptions.tagSelectiveSplitCutOff, tlp);
RemoveDeleteSplittersFromSplitters(tlp, op);
if (op.testOptions.verbose)
{
IList <string> list = new List <string>(op.trainOptions.splitters);
list.Sort();
log.Info("Parent split categories: " + list);
}
}
if (op.trainOptions.selectivePostSplit)
{
// Do all the transformations once just to learn selective splits on annotated categories
ITreeTransformer myTransformer = new TreeAnnotator(tlpParams.HeadFinder(), tlpParams, op);
wholeTreebank = wholeTreebank.Transform(myTransformer);
op.trainOptions.postSplitters = ParentAnnotationStats.GetSplitCategories(wholeTreebank, true, 0, op.trainOptions.selectivePostSplitCutOff, op.trainOptions.tagSelectivePostSplitCutOff, tlp);
if (op.testOptions.verbose)
{
log.Info("Parent post annotation split categories: " + op.trainOptions.postSplitters);
}
}
if (op.trainOptions.hSelSplit)
{
// We run through all the trees once just to gather counts for hSelSplit!
int ptt = op.trainOptions.printTreeTransformations;
op.trainOptions.printTreeTransformations = 0;
binarizer.SetDoSelectiveSplit(false);
foreach (Tree tree in wholeTreebank)
{
trainTransformer.TransformTree(tree);
}
binarizer.SetDoSelectiveSplit(true);
op.trainOptions.printTreeTransformations = ptt;
}
// we've done all the setup now. here's where the train treebank is transformed.
trainTreebank = trainTreebank.Transform(trainTransformer);
if (secondaryTreebank != null)
{
secondaryTreebank = secondaryTreebank.Transform(trainTransformer);
}
if (op.trainOptions.printAnnotatedStateCounts)
{
binarizer.PrintStateCounts();
}
if (op.trainOptions.printAnnotatedRuleCounts)
{
binarizer.PrintRuleCounts();
}
if (tuneTreebank != null)
{
tuneTreebank = tuneTreebank.Transform(trainTransformer);
}
if (op.testOptions.verbose)
{
binarizer.DumpStats();
}
return(new Triple <Treebank, Treebank, Treebank>(trainTreebank, secondaryTreebank, tuneTreebank));
}
public static void Main(string[] args)
{
if (args.Length < minArgs)
{
System.Console.Out.WriteLine(usage);
System.Environment.Exit(-1);
}
// Process command-line options
Properties options = StringUtils.ArgsToProperties(args, optionArgDefinitions);
string fileName = options.GetProperty(string.Empty);
if (fileName == null || fileName.Equals(string.Empty))
{
System.Console.Out.WriteLine(usage);
System.Environment.Exit(-1);
}
Language language = PropertiesUtils.Get(options, "l", Language.English, typeof(Language));
ITreebankLangParserParams tlpp = language.@params;
string encoding = options.GetProperty("e", "UTF-8");
tlpp.SetInputEncoding(encoding);
tlpp.SetOutputEncoding(encoding);
DiskTreebank tb = tlpp.DiskTreebank();
tb.LoadPath(fileName);
// Statistics
ICounter <string> binaryRuleTypes = new ClassicCounter <string>(20000);
IList <int> branchingFactors = new List <int>(20000);
int nTrees = 0;
int nUnaryRules = 0;
int nBinaryRules = 0;
int binaryBranchingFactors = 0;
// Read the treebank
PrintWriter pw = tlpp.Pw();
foreach (Tree tree in tb)
{
if (tree.Value().Equals("ROOT"))
{
tree = tree.FirstChild();
}
++nTrees;
foreach (Tree subTree in tree)
{
if (subTree.IsPhrasal())
{
if (subTree.NumChildren() > 1)
{
++nBinaryRules;
branchingFactors.Add(subTree.NumChildren());
binaryBranchingFactors += subTree.NumChildren();
binaryRuleTypes.IncrementCount(TreeToRuleString(subTree));
}
else
{
++nUnaryRules;
}
}
}
}
double mean = (double)binaryBranchingFactors / (double)nBinaryRules;
System.Console.Out.Printf("#trees:\t%d%n", nTrees);
System.Console.Out.Printf("#binary:\t%d%n", nBinaryRules);
System.Console.Out.Printf("#binary types:\t%d%n", binaryRuleTypes.KeySet().Count);
System.Console.Out.Printf("mean branching:\t%.4f%n", mean);
System.Console.Out.Printf("stddev branching:\t%.4f%n", StandardDeviation(branchingFactors, mean));
System.Console.Out.Printf("rule entropy:\t%.5f%n", Counters.Entropy(binaryRuleTypes));
System.Console.Out.Printf("#unaries:\t%d%n", nUnaryRules);
}
/// <summary>Execute with no arguments for usage.</summary>
public static void Main(string[] args)
{
if (!ValidateCommandLine(args))
{
log.Info(Usage);
System.Environment.Exit(-1);
}
ITreebankLangParserParams tlpp = Language.@params;
PrintWriter pwOut = tlpp.Pw();
Treebank guessTreebank = tlpp.DiskTreebank();
guessTreebank.LoadPath(guessFile);
pwOut.Println("GUESS TREEBANK:");
pwOut.Println(guessTreebank.TextualSummary());
Treebank goldTreebank = tlpp.DiskTreebank();
goldTreebank.LoadPath(goldFile);
pwOut.Println("GOLD TREEBANK:");
pwOut.Println(goldTreebank.TextualSummary());
Edu.Stanford.Nlp.Parser.Metrics.LeafAncestorEval metric = new Edu.Stanford.Nlp.Parser.Metrics.LeafAncestorEval("LeafAncestor");
ITreeTransformer tc = tlpp.Collinizer();
//The evalb ref implementation assigns status for each tree pair as follows:
//
// 0 - Ok (yields match)
// 1 - length mismatch
// 2 - null parse e.g. (()).
//
//In the cases of 1,2, evalb does not include the tree pair in the LP/LR computation.
IEnumerator <Tree> goldItr = goldTreebank.GetEnumerator();
IEnumerator <Tree> guessItr = guessTreebank.GetEnumerator();
int goldLineId = 0;
int guessLineId = 0;
int skippedGuessTrees = 0;
while (guessItr.MoveNext() && goldItr.MoveNext())
{
Tree guessTree = guessItr.Current;
IList <ILabel> guessYield = guessTree.Yield();
guessLineId++;
Tree goldTree = goldItr.Current;
IList <ILabel> goldYield = goldTree.Yield();
goldLineId++;
// Check that we should evaluate this tree
if (goldYield.Count > MaxGoldYield)
{
skippedGuessTrees++;
continue;
}
// Only trees with equal yields can be evaluated
if (goldYield.Count != guessYield.Count)
{
pwOut.Printf("Yield mismatch gold: %d tokens vs. guess: %d tokens (lines: gold %d guess %d)%n", goldYield.Count, guessYield.Count, goldLineId, guessLineId);
skippedGuessTrees++;
continue;
}
Tree evalGuess = tc.TransformTree(guessTree);
Tree evalGold = tc.TransformTree(goldTree);
metric.Evaluate(evalGuess, evalGold, ((Verbose) ? pwOut : null));
}
if (guessItr.MoveNext() || goldItr.MoveNext())
{
System.Console.Error.Printf("Guess/gold files do not have equal lengths (guess: %d gold: %d)%n.", guessLineId, goldLineId);
}
pwOut.Println("================================================================================");
if (skippedGuessTrees != 0)
{
pwOut.Printf("%s %d guess trees%n", "Unable to evaluate", skippedGuessTrees);
}
metric.Display(true, pwOut);
pwOut.Close();
}
/// <summary>Test the parser on a treebank.</summary>
/// <remarks>
/// Test the parser on a treebank. Parses will be written to stdout, and
/// various other information will be written to stderr and stdout,
/// particularly if <code>op.testOptions.verbose</code> is true.
/// </remarks>
/// <param name="testTreebank">The treebank to parse</param>
/// <returns>
/// The labeled precision/recall F<sub>1</sub> (EVALB measure)
/// of the parser on the treebank.
/// </returns>
public virtual double TestOnTreebank(Treebank testTreebank)
{
log.Info("Testing on treebank");
Timing treebankTotalTimer = new Timing();
TreePrint treePrint = op.testOptions.TreePrint(op.tlpParams);
ITreebankLangParserParams tlpParams = op.tlpParams;
ITreebankLanguagePack tlp = op.Langpack();
PrintWriter pwOut;
PrintWriter pwErr;
if (op.testOptions.quietEvaluation)
{
NullOutputStream quiet = new NullOutputStream();
pwOut = tlpParams.Pw(quiet);
pwErr = tlpParams.Pw(quiet);
}
else
{
pwOut = tlpParams.Pw();
pwErr = tlpParams.Pw(System.Console.Error);
}
if (op.testOptions.verbose)
{
pwErr.Print("Testing ");
pwErr.Println(testTreebank.TextualSummary(tlp));
}
if (op.testOptions.evalb)
{
EvalbFormatWriter.InitEVALBfiles(tlpParams);
}
PrintWriter pwFileOut = null;
if (op.testOptions.writeOutputFiles)
{
string fname = op.testOptions.outputFilesPrefix + "." + op.testOptions.outputFilesExtension;
try
{
pwFileOut = op.tlpParams.Pw(new FileOutputStream(fname));
}
catch (IOException ioe)
{
Sharpen.Runtime.PrintStackTrace(ioe);
}
}
PrintWriter pwStats = null;
if (op.testOptions.outputkBestEquivocation != null)
{
try
{
pwStats = op.tlpParams.Pw(new FileOutputStream(op.testOptions.outputkBestEquivocation));
}
catch (IOException ioe)
{
Sharpen.Runtime.PrintStackTrace(ioe);
}
}
if (op.testOptions.testingThreads != 1)
{
MulticoreWrapper <IList <IHasWord>, IParserQuery> wrapper = new MulticoreWrapper <IList <IHasWord>, IParserQuery>(op.testOptions.testingThreads, new ParsingThreadsafeProcessor(pqFactory, pwErr));
LinkedList <Tree> goldTrees = new LinkedList <Tree>();
foreach (Tree goldTree in testTreebank)
{
IList <IHasWord> sentence = GetInputSentence(goldTree);
goldTrees.Add(goldTree);
pwErr.Println("Parsing [len. " + sentence.Count + "]: " + SentenceUtils.ListToString(sentence));
wrapper.Put(sentence);
while (wrapper.Peek())
{
IParserQuery pq = wrapper.Poll();
goldTree = goldTrees.Poll();
ProcessResults(pq, goldTree, pwErr, pwOut, pwFileOut, pwStats, treePrint);
}
}
// for tree iterator
wrapper.Join();
while (wrapper.Peek())
{
IParserQuery pq = wrapper.Poll();
Tree goldTree_1 = goldTrees.Poll();
ProcessResults(pq, goldTree_1, pwErr, pwOut, pwFileOut, pwStats, treePrint);
}
}
else
{
IParserQuery pq = pqFactory.ParserQuery();
foreach (Tree goldTree in testTreebank)
{
IList <CoreLabel> sentence = GetInputSentence(goldTree);
pwErr.Println("Parsing [len. " + sentence.Count + "]: " + SentenceUtils.ListToString(sentence));
pq.ParseAndReport(sentence, pwErr);
ProcessResults(pq, goldTree, pwErr, pwOut, pwFileOut, pwStats, treePrint);
}
}
// for tree iterator
//Done parsing...print the results of the evaluations
treebankTotalTimer.Done("Testing on treebank");
if (op.testOptions.quietEvaluation)
{
pwErr = tlpParams.Pw(System.Console.Error);
}
if (saidMemMessage)
{
ParserUtils.PrintOutOfMemory(pwErr);
}
if (op.testOptions.evalb)
{
EvalbFormatWriter.CloseEVALBfiles();
}
if (numSkippedEvals != 0)
{
pwErr.Printf("Unable to evaluate %d parser hypotheses due to yield mismatch\n", numSkippedEvals);
}
// only created here so we know what parser types are supported...
IParserQuery pq_1 = pqFactory.ParserQuery();
if (summary)
{
if (pcfgLB != null)
{
pcfgLB.Display(false, pwErr);
}
if (pcfgChildSpecific != null)
{
pcfgChildSpecific.Display(false, pwErr);
}
if (pcfgLA != null)
{
pcfgLA.Display(false, pwErr);
}
if (pcfgCB != null)
{
pcfgCB.Display(false, pwErr);
}
if (pcfgDA != null)
{
pcfgDA.Display(false, pwErr);
}
if (pcfgTA != null)
{
pcfgTA.Display(false, pwErr);
}
if (pcfgLL != null && pq_1.GetPCFGParser() != null)
{
pcfgLL.Display(false, pwErr);
}
if (depDA != null)
{
depDA.Display(false, pwErr);
}
if (depTA != null)
{
depTA.Display(false, pwErr);
}
if (depLL != null && pq_1.GetDependencyParser() != null)
{
depLL.Display(false, pwErr);
}
if (factLB != null)
{
factLB.Display(false, pwErr);
}
if (factChildSpecific != null)
{
factChildSpecific.Display(false, pwErr);
}
if (factLA != null)
{
factLA.Display(false, pwErr);
}
if (factCB != null)
{
factCB.Display(false, pwErr);
}
if (factDA != null)
{
factDA.Display(false, pwErr);
}
if (factTA != null)
{
factTA.Display(false, pwErr);
}
if (factLL != null && pq_1.GetFactoredParser() != null)
{
factLL.Display(false, pwErr);
}
if (pcfgCatE != null)
{
pcfgCatE.Display(false, pwErr);
}
foreach (IEval eval in evals)
{
eval.Display(false, pwErr);
}
foreach (BestOfTopKEval eval_1 in topKEvals)
{
eval_1.Display(false, pwErr);
}
}
// these ones only have a display mode, so display if turned on!!
if (pcfgRUO != null)
{
pcfgRUO.Display(true, pwErr);
}
if (pcfgCUO != null)
{
pcfgCUO.Display(true, pwErr);
}
if (tsv)
{
NumberFormat nf = new DecimalFormat("0.00");
pwErr.Println("factF1\tfactDA\tfactEx\tpcfgF1\tdepDA\tfactTA\tnum");
if (factLB != null)
{
pwErr.Print(nf.Format(factLB.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (pq_1.GetDependencyParser() != null && factDA != null)
{
pwErr.Print(nf.Format(factDA.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (factLB != null)
{
pwErr.Print(nf.Format(factLB.GetExactPercent()));
}
pwErr.Print("\t");
if (pcfgLB != null)
{
pwErr.Print(nf.Format(pcfgLB.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (pq_1.GetDependencyParser() != null && depDA != null)
{
pwErr.Print(nf.Format(depDA.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (pq_1.GetPCFGParser() != null && factTA != null)
{
pwErr.Print(nf.Format(factTA.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (factLB != null)
{
pwErr.Print(factLB.GetNum());
}
pwErr.Println();
}
double f1 = 0.0;
if (factLB != null)
{
f1 = factLB.GetEvalbF1();
}
//Close files (if necessary)
if (pwFileOut != null)
{
pwFileOut.Close();
}
if (pwStats != null)
{
pwStats.Close();
}
if (parserQueryEvals != null)
{
foreach (IParserQueryEval parserQueryEval in parserQueryEvals)
{
parserQueryEval.Display(false, pwErr);
}
}
return(f1);
}
/// <param name="args"/>
public static void Main(string[] args)
{
if (args.Length < MinArgs)
{
log.Info(Usage());
System.Environment.Exit(-1);
}
Properties options = StringUtils.ArgsToProperties(args, OptionArgDefs());
bool Verbose = PropertiesUtils.GetBool(options, "v", false);
Language Language = PropertiesUtils.Get(options, "l", Language.English, typeof(Language));
int MaxGoldYield = PropertiesUtils.GetInt(options, "g", int.MaxValue);
int MaxGuessYield = PropertiesUtils.GetInt(options, "y", int.MaxValue);
string[] parsedArgs = options.GetProperty(string.Empty, string.Empty).Split("\\s+");
if (parsedArgs.Length != MinArgs)
{
log.Info(Usage());
System.Environment.Exit(-1);
}
File goldFile = new File(parsedArgs[0]);
File guessFile = new File(parsedArgs[1]);
ITreebankLangParserParams tlpp = Language.@params;
PrintWriter pwOut = tlpp.Pw();
Treebank guessTreebank = tlpp.DiskTreebank();
guessTreebank.LoadPath(guessFile);
pwOut.Println("GUESS TREEBANK:");
pwOut.Println(guessTreebank.TextualSummary());
Treebank goldTreebank = tlpp.DiskTreebank();
goldTreebank.LoadPath(goldFile);
pwOut.Println("GOLD TREEBANK:");
pwOut.Println(goldTreebank.TextualSummary());
Edu.Stanford.Nlp.Parser.Metrics.CollinsDepEval depEval = new Edu.Stanford.Nlp.Parser.Metrics.CollinsDepEval("CollinsDep", true, tlpp.HeadFinder(), tlpp.TreebankLanguagePack().StartSymbol());
ITreeTransformer tc = tlpp.Collinizer();
//PennTreeReader skips over null/malformed parses. So when the yields of the gold/guess trees
//don't match, we need to keep looking for the next gold tree that matches.
//The evalb ref implementation differs slightly as it expects one tree per line. It assigns
//status as follows:
//
// 0 - Ok (yields match)
// 1 - length mismatch
// 2 - null parse e.g. (()).
//
//In the cases of 1,2, evalb does not include the tree pair in the LP/LR computation.
IEnumerator <Tree> goldItr = goldTreebank.GetEnumerator();
int goldLineId = 0;
int skippedGuessTrees = 0;
foreach (Tree guess in guessTreebank)
{
Tree evalGuess = tc.TransformTree(guess);
if (guess.Yield().Count > MaxGuessYield)
{
skippedGuessTrees++;
continue;
}
bool doneEval = false;
while (goldItr.MoveNext() && !doneEval)
{
Tree gold = goldItr.Current;
Tree evalGold = tc.TransformTree(gold);
goldLineId++;
if (gold.Yield().Count > MaxGoldYield)
{
continue;
}
else
{
if (evalGold.Yield().Count != evalGuess.Yield().Count)
{
pwOut.Println("Yield mismatch at gold line " + goldLineId);
skippedGuessTrees++;
break;
}
}
//Default evalb behavior -- skip this guess tree
depEval.Evaluate(evalGuess, evalGold, ((Verbose) ? pwOut : null));
doneEval = true;
}
}
//Move to the next guess parse
pwOut.Println("================================================================================");
if (skippedGuessTrees != 0)
{
pwOut.Printf("%s %d guess trees\n", ((MaxGuessYield < int.MaxValue) ? "Skipped" : "Unable to evaluate"), skippedGuessTrees);
}
depEval.Display(true, pwOut);
pwOut.Close();
}
public static void Main(string[] args)
{
if (args.Length < minArgs)
{
System.Console.Out.WriteLine(usage);
System.Environment.Exit(-1);
}
// Process command-line options
Properties options = StringUtils.ArgsToProperties(args, optionArgDefinitions);
string fileName = options.GetProperty(string.Empty);
if (fileName == null || fileName.Equals(string.Empty))
{
System.Console.Out.WriteLine(usage);
System.Environment.Exit(-1);
}
int maxLen = PropertiesUtils.GetInt(options, "y", int.MaxValue);
bool printTrees = PropertiesUtils.GetBool(options, "p", false);
bool flattenTrees = PropertiesUtils.GetBool(options, "f", false);
bool printPOS = PropertiesUtils.GetBool(options, "a", false);
bool printTnT = PropertiesUtils.GetBool(options, "t", false);
Language language = PropertiesUtils.Get(options, "l", Language.English, typeof(Language));
ITreebankLangParserParams tlpp = language.@params;
string encoding = options.GetProperty("e", "UTF-8");
tlpp.SetInputEncoding(encoding);
tlpp.SetOutputEncoding(encoding);
DiskTreebank tb = tlpp.DiskTreebank();
tb.LoadPath(fileName);
// Read the treebank
PrintWriter pw = tlpp.Pw();
int numTrees = 0;
foreach (Tree tree in tb)
{
if (tree.Yield().Count > maxLen)
{
continue;
}
++numTrees;
if (printTrees)
{
pw.Println(tree.ToString());
}
else
{
if (flattenTrees)
{
pw.Println(SentenceUtils.ListToString(tree.Yield()));
}
else
{
if (printPOS)
{
pw.Println(SentenceUtils.ListToString(tree.PreTerminalYield()));
}
else
{
if (printTnT)
{
IList <CoreLabel> yield = tree.TaggedLabeledYield();
foreach (CoreLabel label in yield)
{
pw.Printf("%s\t%s%n", label.Word(), label.Tag());
}
pw.Println();
}
}
}
}
}
System.Console.Error.Printf("Read %d trees.%n", numTrees);
}
