public static IList <Tree> BinarizeTreebank(Treebank treebank, Options op)
{
TreeBinarizer binarizer = TreeBinarizer.SimpleTreeBinarizer(op.tlpParams.HeadFinder(), op.tlpParams.TreebankLanguagePack());
BasicCategoryTreeTransformer basicTransformer = new BasicCategoryTreeTransformer(op.Langpack());
CompositeTreeTransformer transformer = new CompositeTreeTransformer();
transformer.AddTransformer(binarizer);
transformer.AddTransformer(basicTransformer);
treebank = treebank.Transform(transformer);
IHeadFinder binaryHeadFinder = new BinaryHeadFinder(op.tlpParams.HeadFinder());
IList <Tree> binarizedTrees = Generics.NewArrayList();
foreach (Tree tree in treebank)
{
Edu.Stanford.Nlp.Trees.Trees.ConvertToCoreLabels(tree);
tree.PercolateHeadAnnotations(binaryHeadFinder);
// Index from 1. Tools downstream expect index from 1, so for
// uses internal to the srparser we have to renormalize the
// indices, with the result that here we have to index from 1
tree.IndexLeaves(1, true);
binarizedTrees.Add(tree);
}
return(binarizedTrees);
}
private static Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter GetSegmenterDataFromTreebank(Treebank trainTreebank, Options op, IIndex <string> wordIndex, IIndex <string> tagIndex)
{
System.Console.Out.WriteLine("Currently " + new DateTime());
// printOptions(true, op);
Timing.StartTime();
// setup tree transforms
ITreebankLangParserParams tlpParams = op.tlpParams;
if (op.testOptions.verbose)
{
System.Console.Out.Write("Training ");
System.Console.Out.WriteLine(trainTreebank.TextualSummary());
}
System.Console.Out.Write("Binarizing trees...");
TreeAnnotatorAndBinarizer binarizer;
// initialized below
if (!op.trainOptions.leftToRight)
{
binarizer = new TreeAnnotatorAndBinarizer(tlpParams, op.forceCNF, !op.trainOptions.OutsideFactor(), true, op);
}
else
{
binarizer = new TreeAnnotatorAndBinarizer(tlpParams.HeadFinder(), new LeftHeadFinder(), tlpParams, op.forceCNF, !op.trainOptions.OutsideFactor(), true, op);
}
CollinsPuncTransformer collinsPuncTransformer = null;
if (op.trainOptions.collinsPunc)
{
collinsPuncTransformer = new CollinsPuncTransformer(tlpParams.TreebankLanguagePack());
}
IList <Tree> binaryTrainTrees = new List <Tree>();
// List<Tree> binaryTuneTrees = new ArrayList<Tree>();
if (op.trainOptions.selectiveSplit)
{
op.trainOptions.splitters = ParentAnnotationStats.GetSplitCategories(trainTreebank, true, 0, op.trainOptions.selectiveSplitCutOff, op.trainOptions.tagSelectiveSplitCutOff, tlpParams.TreebankLanguagePack());
if (op.testOptions.verbose)
{
log.Info("Parent split categories: " + op.trainOptions.splitters);
}
}
if (op.trainOptions.selectivePostSplit)
{
ITreeTransformer myTransformer = new TreeAnnotator(tlpParams.HeadFinder(), tlpParams, op);
Treebank annotatedTB = trainTreebank.Transform(myTransformer);
op.trainOptions.postSplitters = ParentAnnotationStats.GetSplitCategories(annotatedTB, true, 0, op.trainOptions.selectivePostSplitCutOff, op.trainOptions.tagSelectivePostSplitCutOff, tlpParams.TreebankLanguagePack());
if (op.testOptions.verbose)
{
log.Info("Parent post annotation split categories: " + op.trainOptions.postSplitters);
}
}
if (op.trainOptions.hSelSplit)
{
binarizer.SetDoSelectiveSplit(false);
foreach (Tree tree in trainTreebank)
{
if (op.trainOptions.collinsPunc)
{
tree = collinsPuncTransformer.TransformTree(tree);
}
tree = binarizer.TransformTree(tree);
}
binarizer.SetDoSelectiveSplit(true);
}
foreach (Tree tree_1 in trainTreebank)
{
if (op.trainOptions.collinsPunc)
{
tree_1 = collinsPuncTransformer.TransformTree(tree_1);
}
tree_1 = binarizer.TransformTree(tree_1);
binaryTrainTrees.Add(tree_1);
}
Timing.Tick("done.");
if (op.testOptions.verbose)
{
binarizer.DumpStats();
}
System.Console.Out.Write("Extracting Lexicon...");
Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter clex = (Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter)op.tlpParams.Lex(op, wordIndex, tagIndex);
clex.InitializeTraining(binaryTrainTrees.Count);
clex.Train(binaryTrainTrees);
clex.FinishTraining();
Timing.Tick("done.");
return(clex);
}
/// <summary>
/// An example command line for training a new parser:
/// <br />
/// nohup java -mx6g edu.stanford.nlp.parser.dvparser.DVParser -cachedTrees /scr/nlp/data/dvparser/wsj/cached.wsj.train.simple.ser.gz -train -testTreebank /afs/ir/data/linguistic-data/Treebank/3/parsed/mrg/wsj/22 2200-2219 -debugOutputFrequency 400 -nofilter -trainingThreads 5 -parser /u/nlp/data/lexparser/wsjPCFG.nocompact.simple.ser.gz -trainingIterations 40 -batchSize 25 -model /scr/nlp/data/dvparser/wsj/wsj.combine.v2.ser.gz -unkWord "*UNK*" -dvCombineCategories > /scr/nlp/data/dvparser/wsj/wsj.combine.v2.out 2>&1 &
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
public static void Main(string[] args)
{
if (args.Length == 0)
{
Help();
System.Environment.Exit(2);
}
log.Info("Running DVParser with arguments:");
foreach (string arg in args)
{
log.Info(" " + arg);
}
log.Info();
string parserPath = null;
string trainTreebankPath = null;
IFileFilter trainTreebankFilter = null;
string cachedTrainTreesPath = null;
bool runGradientCheck = false;
bool runTraining = false;
string testTreebankPath = null;
IFileFilter testTreebankFilter = null;
string initialModelPath = null;
string modelPath = null;
bool filter = true;
string resultsRecordPath = null;
IList <string> unusedArgs = new List <string>();
// These parameters can be null or 0 if the model was not
// serialized with the new parameters. Setting the options at the
// command line will override these defaults.
// TODO: if/when we integrate back into the main branch and
// rebuild models, we can get rid of this
IList <string> argsWithDefaults = new List <string>(Arrays.AsList(new string[] { "-wordVectorFile", Options.LexOptions.DefaultWordVectorFile, "-dvKBest", int.ToString(TrainOptions.DefaultKBest), "-batchSize", int.ToString(TrainOptions.DefaultBatchSize
), "-trainingIterations", int.ToString(TrainOptions.DefaultTrainingIterations), "-qnIterationsPerBatch", int.ToString(TrainOptions.DefaultQnIterationsPerBatch), "-regCost", double.ToString(TrainOptions.DefaultRegcost), "-learningRate", double
.ToString(TrainOptions.DefaultLearningRate), "-deltaMargin", double.ToString(TrainOptions.DefaultDeltaMargin), "-unknownNumberVector", "-unknownDashedWordVectors", "-unknownCapsVector", "-unknownchinesepercentvector", "-unknownchinesenumbervector"
, "-unknownchineseyearvector", "-unkWord", "*UNK*", "-transformMatrixType", "DIAGONAL", "-scalingForInit", double.ToString(TrainOptions.DefaultScalingForInit), "-trainWordVectors" }));
Sharpen.Collections.AddAll(argsWithDefaults, Arrays.AsList(args));
args = Sharpen.Collections.ToArray(argsWithDefaults, new string[argsWithDefaults.Count]);
for (int argIndex = 0; argIndex < args.Length;)
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-parser"))
{
parserPath = args[argIndex + 1];
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-testTreebank"))
{
Pair <string, IFileFilter> treebankDescription = ArgUtils.GetTreebankDescription(args, argIndex, "-testTreebank");
argIndex = argIndex + ArgUtils.NumSubArgs(args, argIndex) + 1;
testTreebankPath = treebankDescription.First();
testTreebankFilter = treebankDescription.Second();
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-treebank"))
{
Pair <string, IFileFilter> treebankDescription = ArgUtils.GetTreebankDescription(args, argIndex, "-treebank");
argIndex = argIndex + ArgUtils.NumSubArgs(args, argIndex) + 1;
trainTreebankPath = treebankDescription.First();
trainTreebankFilter = treebankDescription.Second();
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-cachedTrees"))
{
cachedTrainTreesPath = args[argIndex + 1];
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-runGradientCheck"))
{
runGradientCheck = true;
argIndex++;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-train"))
{
runTraining = true;
argIndex++;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-model"))
{
modelPath = args[argIndex + 1];
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-nofilter"))
{
filter = false;
argIndex++;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-continueTraining"))
{
runTraining = true;
filter = false;
initialModelPath = args[argIndex + 1];
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-resultsRecord"))
{
resultsRecordPath = args[argIndex + 1];
argIndex += 2;
}
else
{
unusedArgs.Add(args[argIndex++]);
}
}
}
}
}
}
}
}
}
}
}
if (parserPath == null && modelPath == null)
{
throw new ArgumentException("Must supply either a base parser model with -parser or a serialized DVParser with -model");
}
if (!runTraining && modelPath == null && !runGradientCheck)
{
throw new ArgumentException("Need to either train a new model, run the gradient check or specify a model to load with -model");
}
string[] newArgs = Sharpen.Collections.ToArray(unusedArgs, new string[unusedArgs.Count]);
Edu.Stanford.Nlp.Parser.Dvparser.DVParser dvparser = null;
LexicalizedParser lexparser = null;
if (initialModelPath != null)
{
lexparser = ((LexicalizedParser)LexicalizedParser.LoadModel(initialModelPath, newArgs));
DVModel model = GetModelFromLexicalizedParser(lexparser);
dvparser = new Edu.Stanford.Nlp.Parser.Dvparser.DVParser(model, lexparser);
}
else
{
if (runTraining || runGradientCheck)
{
lexparser = ((LexicalizedParser)LexicalizedParser.LoadModel(parserPath, newArgs));
dvparser = new Edu.Stanford.Nlp.Parser.Dvparser.DVParser(lexparser);
}
else
{
if (modelPath != null)
{
lexparser = ((LexicalizedParser)LexicalizedParser.LoadModel(modelPath, newArgs));
DVModel model = GetModelFromLexicalizedParser(lexparser);
dvparser = new Edu.Stanford.Nlp.Parser.Dvparser.DVParser(model, lexparser);
}
}
}
IList <Tree> trainSentences = new List <Tree>();
IdentityHashMap <Tree, byte[]> trainCompressedParses = Generics.NewIdentityHashMap();
if (cachedTrainTreesPath != null)
{
foreach (string path in cachedTrainTreesPath.Split(","))
{
IList <Pair <Tree, byte[]> > cache = IOUtils.ReadObjectFromFile(path);
foreach (Pair <Tree, byte[]> pair in cache)
{
trainSentences.Add(pair.First());
trainCompressedParses[pair.First()] = pair.Second();
}
log.Info("Read in " + cache.Count + " trees from " + path);
}
}
if (trainTreebankPath != null)
{
// TODO: make the transformer a member of the model?
ITreeTransformer transformer = BuildTrainTransformer(dvparser.GetOp());
Treebank treebank = dvparser.GetOp().tlpParams.MemoryTreebank();
treebank.LoadPath(trainTreebankPath, trainTreebankFilter);
treebank = treebank.Transform(transformer);
log.Info("Read in " + treebank.Count + " trees from " + trainTreebankPath);
CacheParseHypotheses cacher = new CacheParseHypotheses(dvparser.parser);
CacheParseHypotheses.CacheProcessor processor = new CacheParseHypotheses.CacheProcessor(cacher, lexparser, dvparser.op.trainOptions.dvKBest, transformer);
foreach (Tree tree in treebank)
{
trainSentences.Add(tree);
trainCompressedParses[tree] = processor.Process(tree).second;
}
//System.out.println(tree);
log.Info("Finished parsing " + treebank.Count + " trees, getting " + dvparser.op.trainOptions.dvKBest + " hypotheses each");
}
if ((runTraining || runGradientCheck) && filter)
{
log.Info("Filtering rules for the given training set");
dvparser.dvModel.SetRulesForTrainingSet(trainSentences, trainCompressedParses);
log.Info("Done filtering rules; " + dvparser.dvModel.numBinaryMatrices + " binary matrices, " + dvparser.dvModel.numUnaryMatrices + " unary matrices, " + dvparser.dvModel.wordVectors.Count + " word vectors");
}
//dvparser.dvModel.printAllMatrices();
Treebank testTreebank = null;
if (testTreebankPath != null)
{
log.Info("Reading in trees from " + testTreebankPath);
if (testTreebankFilter != null)
{
log.Info("Filtering on " + testTreebankFilter);
}
testTreebank = dvparser.GetOp().tlpParams.MemoryTreebank();
testTreebank.LoadPath(testTreebankPath, testTreebankFilter);
log.Info("Read in " + testTreebank.Count + " trees for testing");
}
// runGradientCheck= true;
if (runGradientCheck)
{
log.Info("Running gradient check on " + trainSentences.Count + " trees");
dvparser.RunGradientCheck(trainSentences, trainCompressedParses);
}
if (runTraining)
{
log.Info("Training the RNN parser");
log.Info("Current train options: " + dvparser.GetOp().trainOptions);
dvparser.Train(trainSentences, trainCompressedParses, testTreebank, modelPath, resultsRecordPath);
if (modelPath != null)
{
dvparser.SaveModel(modelPath);
}
}
if (testTreebankPath != null)
{
EvaluateTreebank evaluator = new EvaluateTreebank(dvparser.AttachModelToLexicalizedParser());
evaluator.TestOnTreebank(testTreebank);
}
log.Info("Successfully ran DVParser");
}
/// <returns>A Triple of binaryTrainTreebank, binarySecondaryTreebank, binaryTuneTreebank.</returns>
public static Triple <Treebank, Treebank, Treebank> GetAnnotatedBinaryTreebankFromTreebank(Treebank trainTreebank, Treebank secondaryTreebank, Treebank tuneTreebank, Options op)
{
// setup tree transforms
ITreebankLangParserParams tlpParams = op.tlpParams;
ITreebankLanguagePack tlp = tlpParams.TreebankLanguagePack();
if (op.testOptions.verbose)
{
PrintWriter pwErr = tlpParams.Pw(System.Console.Error);
pwErr.Print("Training ");
pwErr.Println(trainTreebank.TextualSummary(tlp));
if (secondaryTreebank != null)
{
pwErr.Print("Secondary training ");
pwErr.Println(secondaryTreebank.TextualSummary(tlp));
}
}
CompositeTreeTransformer trainTransformer = new CompositeTreeTransformer();
if (op.trainOptions.preTransformer != null)
{
trainTransformer.AddTransformer(op.trainOptions.preTransformer);
}
if (op.trainOptions.collinsPunc)
{
CollinsPuncTransformer collinsPuncTransformer = new CollinsPuncTransformer(tlp);
trainTransformer.AddTransformer(collinsPuncTransformer);
}
log.Info("Binarizing trees...");
Edu.Stanford.Nlp.Parser.Lexparser.TreeAnnotatorAndBinarizer binarizer;
if (!op.trainOptions.leftToRight)
{
binarizer = new Edu.Stanford.Nlp.Parser.Lexparser.TreeAnnotatorAndBinarizer(tlpParams, op.forceCNF, !op.trainOptions.OutsideFactor(), !op.trainOptions.predictSplits, op);
}
else
{
binarizer = new Edu.Stanford.Nlp.Parser.Lexparser.TreeAnnotatorAndBinarizer(tlpParams.HeadFinder(), new LeftHeadFinder(), tlpParams, op.forceCNF, !op.trainOptions.OutsideFactor(), !op.trainOptions.predictSplits, op);
}
trainTransformer.AddTransformer(binarizer);
if (op.wordFunction != null)
{
ITreeTransformer wordFunctionTransformer = new TreeLeafLabelTransformer(op.wordFunction);
trainTransformer.AddTransformer(wordFunctionTransformer);
}
Treebank wholeTreebank;
if (secondaryTreebank == null)
{
wholeTreebank = trainTreebank;
}
else
{
wholeTreebank = new CompositeTreebank(trainTreebank, secondaryTreebank);
}
if (op.trainOptions.selectiveSplit)
{
op.trainOptions.splitters = ParentAnnotationStats.GetSplitCategories(wholeTreebank, op.trainOptions.tagSelectiveSplit, 0, op.trainOptions.selectiveSplitCutOff, op.trainOptions.tagSelectiveSplitCutOff, tlp);
RemoveDeleteSplittersFromSplitters(tlp, op);
if (op.testOptions.verbose)
{
IList <string> list = new List <string>(op.trainOptions.splitters);
list.Sort();
log.Info("Parent split categories: " + list);
}
}
if (op.trainOptions.selectivePostSplit)
{
// Do all the transformations once just to learn selective splits on annotated categories
ITreeTransformer myTransformer = new TreeAnnotator(tlpParams.HeadFinder(), tlpParams, op);
wholeTreebank = wholeTreebank.Transform(myTransformer);
op.trainOptions.postSplitters = ParentAnnotationStats.GetSplitCategories(wholeTreebank, true, 0, op.trainOptions.selectivePostSplitCutOff, op.trainOptions.tagSelectivePostSplitCutOff, tlp);
if (op.testOptions.verbose)
{
log.Info("Parent post annotation split categories: " + op.trainOptions.postSplitters);
}
}
if (op.trainOptions.hSelSplit)
{
// We run through all the trees once just to gather counts for hSelSplit!
int ptt = op.trainOptions.printTreeTransformations;
op.trainOptions.printTreeTransformations = 0;
binarizer.SetDoSelectiveSplit(false);
foreach (Tree tree in wholeTreebank)
{
trainTransformer.TransformTree(tree);
}
binarizer.SetDoSelectiveSplit(true);
op.trainOptions.printTreeTransformations = ptt;
}
// we've done all the setup now. here's where the train treebank is transformed.
trainTreebank = trainTreebank.Transform(trainTransformer);
if (secondaryTreebank != null)
{
secondaryTreebank = secondaryTreebank.Transform(trainTransformer);
}
if (op.trainOptions.printAnnotatedStateCounts)
{
binarizer.PrintStateCounts();
}
if (op.trainOptions.printAnnotatedRuleCounts)
{
binarizer.PrintRuleCounts();
}
if (tuneTreebank != null)
{
tuneTreebank = tuneTreebank.Transform(trainTransformer);
}
if (op.testOptions.verbose)
{
binarizer.DumpStats();
}
return(new Triple <Treebank, Treebank, Treebank>(trainTreebank, secondaryTreebank, tuneTreebank));
}
/// <summary>
/// An example of a command line is
/// <br />
/// java -mx1g edu.stanford.nlp.parser.dvparser.CacheParseHypotheses -model /scr/horatio/dvparser/wsjPCFG.nocompact.simple.ser.gz -output cached9.simple.ser.gz -treebank /afs/ir/data/linguistic-data/Treebank/3/parsed/mrg/wsj 200-202
/// <br />
/// java -mx4g edu.stanford.nlp.parser.dvparser.CacheParseHypotheses -model ~/scr/dvparser/wsjPCFG.nocompact.simple.ser.gz -output cached.train.simple.ser.gz -treebank /afs/ir/data/linguistic-data/Treebank/3/parsed/mrg/wsj 200-2199 -numThreads 6
/// <br />
/// java -mx4g edu.stanford.nlp.parser.dvparser.CacheParseHypotheses -model ~/scr/dvparser/chinese/xinhuaPCFG.ser.gz -output cached.xinhua.train.ser.gz -treebank /afs/ir/data/linguistic-data/Chinese-Treebank/6/data/utf8/bracketed 026-270,301-499,600-999
/// </summary>
/// <exception cref="System.IO.IOException"/>
public static void Main(string[] args)
{
string parserModel = null;
string output = null;
IList <Pair <string, IFileFilter> > treebanks = Generics.NewArrayList();
int dvKBest = 200;
int numThreads = 1;
for (int argIndex = 0; argIndex < args.Length;)
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-dvKBest"))
{
dvKBest = System.Convert.ToInt32(args[argIndex + 1]);
argIndex += 2;
continue;
}
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-parser") || args[argIndex].Equals("-model"))
{
parserModel = args[argIndex + 1];
argIndex += 2;
continue;
}
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-output"))
{
output = args[argIndex + 1];
argIndex += 2;
continue;
}
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-treebank"))
{
Pair <string, IFileFilter> treebankDescription = ArgUtils.GetTreebankDescription(args, argIndex, "-treebank");
argIndex = argIndex + ArgUtils.NumSubArgs(args, argIndex) + 1;
treebanks.Add(treebankDescription);
continue;
}
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-numThreads"))
{
numThreads = System.Convert.ToInt32(args[argIndex + 1]);
argIndex += 2;
continue;
}
throw new ArgumentException("Unknown argument " + args[argIndex]);
}
if (parserModel == null)
{
throw new ArgumentException("Need to supply a parser model with -model");
}
if (output == null)
{
throw new ArgumentException("Need to supply an output filename with -output");
}
if (treebanks.IsEmpty())
{
throw new ArgumentException("Need to supply a treebank with -treebank");
}
log.Info("Writing output to " + output);
log.Info("Loading parser model " + parserModel);
log.Info("Writing " + dvKBest + " hypothesis trees for each tree");
LexicalizedParser parser = ((LexicalizedParser)LexicalizedParser.LoadModel(parserModel, "-dvKBest", int.ToString(dvKBest)));
CacheParseHypotheses cacher = new CacheParseHypotheses(parser);
ITreeTransformer transformer = DVParser.BuildTrainTransformer(parser.GetOp());
IList <Tree> sentences = new List <Tree>();
foreach (Pair <string, IFileFilter> description in treebanks)
{
log.Info("Reading trees from " + description.first);
Treebank treebank = parser.GetOp().tlpParams.MemoryTreebank();
treebank.LoadPath(description.first, description.second);
treebank = treebank.Transform(transformer);
Sharpen.Collections.AddAll(sentences, treebank);
}
log.Info("Processing " + sentences.Count + " trees");
IList <Pair <Tree, byte[]> > cache = Generics.NewArrayList();
transformer = new SynchronizedTreeTransformer(transformer);
MulticoreWrapper <Tree, Pair <Tree, byte[]> > wrapper = new MulticoreWrapper <Tree, Pair <Tree, byte[]> >(numThreads, new CacheParseHypotheses.CacheProcessor(cacher, parser, dvKBest, transformer));
foreach (Tree tree in sentences)
{
wrapper.Put(tree);
while (wrapper.Peek())
{
cache.Add(wrapper.Poll());
if (cache.Count % 10 == 0)
{
System.Console.Out.WriteLine("Processed " + cache.Count + " trees");
}
}
}
wrapper.Join();
while (wrapper.Peek())
{
cache.Add(wrapper.Poll());
if (cache.Count % 10 == 0)
{
System.Console.Out.WriteLine("Processed " + cache.Count + " trees");
}
}
System.Console.Out.WriteLine("Finished processing " + cache.Count + " trees");
IOUtils.WriteObjectToFile(cache, output);
}
/* some documentation for Roger's convenience
* {pcfg,dep,combo}{PE,DE,TE} are precision/dep/tagging evals for the models
*
* parser is the PCFG parser
* dparser is the dependency parser
* bparser is the combining parser
*
* during testing:
* tree is the test tree (gold tree)
* binaryTree is the gold tree binarized
* tree2b is the best PCFG paser, binarized
* tree2 is the best PCFG parse (debinarized)
* tree3 is the dependency parse, binarized
* tree3db is the dependency parser, debinarized
* tree4 is the best combo parse, binarized and then debinarized
* tree4b is the best combo parse, binarized
*/
public static void Main(string[] args)
{
Options op = new Options(new EnglishTreebankParserParams());
// op.tlpParams may be changed to something else later, so don't use it till
// after options are parsed.
StringUtils.LogInvocationString(log, args);
string path = "/u/nlp/stuff/corpora/Treebank3/parsed/mrg/wsj";
int trainLow = 200;
int trainHigh = 2199;
int testLow = 2200;
int testHigh = 2219;
string serializeFile = null;
int i = 0;
while (i < args.Length && args[i].StartsWith("-"))
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-path") && (i + 1 < args.Length))
{
path = args[i + 1];
i += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-train") && (i + 2 < args.Length))
{
trainLow = System.Convert.ToInt32(args[i + 1]);
trainHigh = System.Convert.ToInt32(args[i + 2]);
i += 3;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-test") && (i + 2 < args.Length))
{
testLow = System.Convert.ToInt32(args[i + 1]);
testHigh = System.Convert.ToInt32(args[i + 2]);
i += 3;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-serialize") && (i + 1 < args.Length))
{
serializeFile = args[i + 1];
i += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-tLPP") && (i + 1 < args.Length))
{
try
{
op.tlpParams = (ITreebankLangParserParams)System.Activator.CreateInstance(Sharpen.Runtime.GetType(args[i + 1]));
}
catch (TypeLoadException e)
{
log.Info("Class not found: " + args[i + 1]);
throw new Exception(e);
}
catch (InstantiationException e)
{
log.Info("Couldn't instantiate: " + args[i + 1] + ": " + e.ToString());
throw new Exception(e);
}
catch (MemberAccessException e)
{
log.Info("illegal access" + e);
throw new Exception(e);
}
i += 2;
}
else
{
if (args[i].Equals("-encoding"))
{
// sets encoding for TreebankLangParserParams
op.tlpParams.SetInputEncoding(args[i + 1]);
op.tlpParams.SetOutputEncoding(args[i + 1]);
i += 2;
}
else
{
i = op.SetOptionOrWarn(args, i);
}
}
}
}
}
}
}
// System.out.println(tlpParams.getClass());
ITreebankLanguagePack tlp = op.tlpParams.TreebankLanguagePack();
op.trainOptions.sisterSplitters = Generics.NewHashSet(Arrays.AsList(op.tlpParams.SisterSplitters()));
// BinarizerFactory.TreeAnnotator.setTreebankLang(tlpParams);
PrintWriter pw = op.tlpParams.Pw();
op.testOptions.Display();
op.trainOptions.Display();
op.Display();
op.tlpParams.Display();
// setup tree transforms
Treebank trainTreebank = op.tlpParams.MemoryTreebank();
MemoryTreebank testTreebank = op.tlpParams.TestMemoryTreebank();
// Treebank blippTreebank = ((EnglishTreebankParserParams) tlpParams).diskTreebank();
// String blippPath = "/afs/ir.stanford.edu/data/linguistic-data/BLLIP-WSJ/";
// blippTreebank.loadPath(blippPath, "", true);
Timing.StartTime();
log.Info("Reading trees...");
testTreebank.LoadPath(path, new NumberRangeFileFilter(testLow, testHigh, true));
if (op.testOptions.increasingLength)
{
testTreebank.Sort(new TreeLengthComparator());
}
trainTreebank.LoadPath(path, new NumberRangeFileFilter(trainLow, trainHigh, true));
Timing.Tick("done.");
log.Info("Binarizing trees...");
TreeAnnotatorAndBinarizer binarizer;
if (!op.trainOptions.leftToRight)
{
binarizer = new TreeAnnotatorAndBinarizer(op.tlpParams, op.forceCNF, !op.trainOptions.OutsideFactor(), true, op);
}
else
{
binarizer = new TreeAnnotatorAndBinarizer(op.tlpParams.HeadFinder(), new LeftHeadFinder(), op.tlpParams, op.forceCNF, !op.trainOptions.OutsideFactor(), true, op);
}
CollinsPuncTransformer collinsPuncTransformer = null;
if (op.trainOptions.collinsPunc)
{
collinsPuncTransformer = new CollinsPuncTransformer(tlp);
}
ITreeTransformer debinarizer = new Debinarizer(op.forceCNF);
IList <Tree> binaryTrainTrees = new List <Tree>();
if (op.trainOptions.selectiveSplit)
{
op.trainOptions.splitters = ParentAnnotationStats.GetSplitCategories(trainTreebank, op.trainOptions.tagSelectiveSplit, 0, op.trainOptions.selectiveSplitCutOff, op.trainOptions.tagSelectiveSplitCutOff, op.tlpParams.TreebankLanguagePack());
if (op.trainOptions.deleteSplitters != null)
{
IList <string> deleted = new List <string>();
foreach (string del in op.trainOptions.deleteSplitters)
{
string baseDel = tlp.BasicCategory(del);
bool checkBasic = del.Equals(baseDel);
for (IEnumerator <string> it = op.trainOptions.splitters.GetEnumerator(); it.MoveNext();)
{
string elem = it.Current;
string baseElem = tlp.BasicCategory(elem);
bool delStr = checkBasic && baseElem.Equals(baseDel) || elem.Equals(del);
if (delStr)
{
it.Remove();
deleted.Add(elem);
}
}
}
log.Info("Removed from vertical splitters: " + deleted);
}
}
if (op.trainOptions.selectivePostSplit)
{
ITreeTransformer myTransformer = new TreeAnnotator(op.tlpParams.HeadFinder(), op.tlpParams, op);
Treebank annotatedTB = trainTreebank.Transform(myTransformer);
op.trainOptions.postSplitters = ParentAnnotationStats.GetSplitCategories(annotatedTB, true, 0, op.trainOptions.selectivePostSplitCutOff, op.trainOptions.tagSelectivePostSplitCutOff, op.tlpParams.TreebankLanguagePack());
}
if (op.trainOptions.hSelSplit)
{
binarizer.SetDoSelectiveSplit(false);
foreach (Tree tree in trainTreebank)
{
if (op.trainOptions.collinsPunc)
{
tree = collinsPuncTransformer.TransformTree(tree);
}
//tree.pennPrint(tlpParams.pw());
tree = binarizer.TransformTree(tree);
}
//binaryTrainTrees.add(tree);
binarizer.SetDoSelectiveSplit(true);
}
foreach (Tree tree_1 in trainTreebank)
{
if (op.trainOptions.collinsPunc)
{
tree_1 = collinsPuncTransformer.TransformTree(tree_1);
}
tree_1 = binarizer.TransformTree(tree_1);
binaryTrainTrees.Add(tree_1);
}
if (op.testOptions.verbose)
{
binarizer.DumpStats();
}
IList <Tree> binaryTestTrees = new List <Tree>();
foreach (Tree tree_2 in testTreebank)
{
if (op.trainOptions.collinsPunc)
{
tree_2 = collinsPuncTransformer.TransformTree(tree_2);
}
tree_2 = binarizer.TransformTree(tree_2);
binaryTestTrees.Add(tree_2);
}
Timing.Tick("done.");
// binarization
BinaryGrammar bg = null;
UnaryGrammar ug = null;
IDependencyGrammar dg = null;
// DependencyGrammar dgBLIPP = null;
ILexicon lex = null;
IIndex <string> stateIndex = new HashIndex <string>();
// extract grammars
IExtractor <Pair <UnaryGrammar, BinaryGrammar> > bgExtractor = new BinaryGrammarExtractor(op, stateIndex);
//Extractor bgExtractor = new SmoothedBinaryGrammarExtractor();//new BinaryGrammarExtractor();
// Extractor lexExtractor = new LexiconExtractor();
//Extractor dgExtractor = new DependencyMemGrammarExtractor();
if (op.doPCFG)
{
log.Info("Extracting PCFG...");
Pair <UnaryGrammar, BinaryGrammar> bgug = null;
if (op.trainOptions.cheatPCFG)
{
IList <Tree> allTrees = new List <Tree>(binaryTrainTrees);
Sharpen.Collections.AddAll(allTrees, binaryTestTrees);
bgug = bgExtractor.Extract(allTrees);
}
else
{
bgug = bgExtractor.Extract(binaryTrainTrees);
}
bg = bgug.second;
bg.SplitRules();
ug = bgug.first;
ug.PurgeRules();
Timing.Tick("done.");
}
log.Info("Extracting Lexicon...");
IIndex <string> wordIndex = new HashIndex <string>();
IIndex <string> tagIndex = new HashIndex <string>();
lex = op.tlpParams.Lex(op, wordIndex, tagIndex);
lex.InitializeTraining(binaryTrainTrees.Count);
lex.Train(binaryTrainTrees);
lex.FinishTraining();
Timing.Tick("done.");
if (op.doDep)
{
log.Info("Extracting Dependencies...");
binaryTrainTrees.Clear();
IExtractor <IDependencyGrammar> dgExtractor = new MLEDependencyGrammarExtractor(op, wordIndex, tagIndex);
// dgBLIPP = (DependencyGrammar) dgExtractor.extract(new ConcatenationIterator(trainTreebank.iterator(),blippTreebank.iterator()),new TransformTreeDependency(tlpParams,true));
// DependencyGrammar dg1 = dgExtractor.extract(trainTreebank.iterator(), new TransformTreeDependency(op.tlpParams, true));
//dgBLIPP=(DependencyGrammar)dgExtractor.extract(blippTreebank.iterator(),new TransformTreeDependency(tlpParams));
//dg = (DependencyGrammar) dgExtractor.extract(new ConcatenationIterator(trainTreebank.iterator(),blippTreebank.iterator()),new TransformTreeDependency(tlpParams));
// dg=new DependencyGrammarCombination(dg1,dgBLIPP,2);
dg = dgExtractor.Extract(binaryTrainTrees);
//uses information whether the words are known or not, discards unknown words
Timing.Tick("done.");
//System.out.print("Extracting Unknown Word Model...");
//UnknownWordModel uwm = (UnknownWordModel)uwmExtractor.extract(binaryTrainTrees);
//Timing.tick("done.");
System.Console.Out.Write("Tuning Dependency Model...");
dg.Tune(binaryTestTrees);
//System.out.println("TUNE DEPS: "+tuneDeps);
Timing.Tick("done.");
}
BinaryGrammar boundBG = bg;
UnaryGrammar boundUG = ug;
IGrammarProjection gp = new NullGrammarProjection(bg, ug);
// serialization
if (serializeFile != null)
{
log.Info("Serializing parser...");
LexicalizedParser parser = new LexicalizedParser(lex, bg, ug, dg, stateIndex, wordIndex, tagIndex, op);
parser.SaveParserToSerialized(serializeFile);
Timing.Tick("done.");
}
// test: pcfg-parse and output
ExhaustivePCFGParser parser_1 = null;
if (op.doPCFG)
{
parser_1 = new ExhaustivePCFGParser(boundBG, boundUG, lex, op, stateIndex, wordIndex, tagIndex);
}
ExhaustiveDependencyParser dparser = ((op.doDep && !op.testOptions.useFastFactored) ? new ExhaustiveDependencyParser(dg, lex, op, wordIndex, tagIndex) : null);
IScorer scorer = (op.doPCFG ? new TwinScorer(new ProjectionScorer(parser_1, gp, op), dparser) : null);
//Scorer scorer = parser;
BiLexPCFGParser bparser = null;
if (op.doPCFG && op.doDep)
{
bparser = (op.testOptions.useN5) ? new BiLexPCFGParser.N5BiLexPCFGParser(scorer, parser_1, dparser, bg, ug, dg, lex, op, gp, stateIndex, wordIndex, tagIndex) : new BiLexPCFGParser(scorer, parser_1, dparser, bg, ug, dg, lex, op, gp, stateIndex
, wordIndex, tagIndex);
}
Evalb pcfgPE = new Evalb("pcfg PE", true);
Evalb comboPE = new Evalb("combo PE", true);
AbstractEval pcfgCB = new Evalb.CBEval("pcfg CB", true);
AbstractEval pcfgTE = new TaggingEval("pcfg TE");
AbstractEval comboTE = new TaggingEval("combo TE");
AbstractEval pcfgTEnoPunct = new TaggingEval("pcfg nopunct TE");
AbstractEval comboTEnoPunct = new TaggingEval("combo nopunct TE");
AbstractEval depTE = new TaggingEval("depnd TE");
AbstractEval depDE = new UnlabeledAttachmentEval("depnd DE", true, null, tlp.PunctuationWordRejectFilter());
AbstractEval comboDE = new UnlabeledAttachmentEval("combo DE", true, null, tlp.PunctuationWordRejectFilter());
if (op.testOptions.evalb)
{
EvalbFormatWriter.InitEVALBfiles(op.tlpParams);
}
// int[] countByLength = new int[op.testOptions.maxLength+1];
// Use a reflection ruse, so one can run this without needing the
// tagger. Using a function rather than a MaxentTagger means we
// can distribute a version of the parser that doesn't include the
// entire tagger.
IFunction <IList <IHasWord>, List <TaggedWord> > tagger = null;
if (op.testOptions.preTag)
{
try
{
Type[] argsClass = new Type[] { typeof(string) };
object[] arguments = new object[] { op.testOptions.taggerSerializedFile };
tagger = (IFunction <IList <IHasWord>, List <TaggedWord> >)Sharpen.Runtime.GetType("edu.stanford.nlp.tagger.maxent.MaxentTagger").GetConstructor(argsClass).NewInstance(arguments);
}
catch (Exception e)
{
log.Info(e);
log.Info("Warning: No pretagging of sentences will be done.");
}
}
for (int tNum = 0; tNum < ttSize; tNum++)
{
Tree tree = testTreebank[tNum];
int testTreeLen = tree_2.Yield().Count;
if (testTreeLen > op.testOptions.maxLength)
{
continue;
}
Tree binaryTree = binaryTestTrees[tNum];
// countByLength[testTreeLen]++;
System.Console.Out.WriteLine("-------------------------------------");
System.Console.Out.WriteLine("Number: " + (tNum + 1));
System.Console.Out.WriteLine("Length: " + testTreeLen);
//tree.pennPrint(pw);
// System.out.println("XXXX The binary tree is");
// binaryTree.pennPrint(pw);
//System.out.println("Here are the tags in the lexicon:");
//System.out.println(lex.showTags());
//System.out.println("Here's the tagnumberer:");
//System.out.println(Numberer.getGlobalNumberer("tags").toString());
long timeMil1 = Runtime.CurrentTimeMillis();
Timing.Tick("Starting parse.");
if (op.doPCFG)
{
//log.info(op.testOptions.forceTags);
if (op.testOptions.forceTags)
{
if (tagger != null)
{
//System.out.println("Using a tagger to set tags");
//System.out.println("Tagged sentence as: " + tagger.processSentence(cutLast(wordify(binaryTree.yield()))).toString(false));
parser_1.Parse(AddLast(tagger.Apply(CutLast(Wordify(binaryTree.Yield())))));
}
else
{
//System.out.println("Forcing tags to match input.");
parser_1.Parse(CleanTags(binaryTree.TaggedYield(), tlp));
}
}
else
{
// System.out.println("XXXX Parsing " + binaryTree.yield());
parser_1.Parse(binaryTree.YieldHasWord());
}
}
//Timing.tick("Done with pcfg phase.");
if (op.doDep)
{
dparser.Parse(binaryTree.YieldHasWord());
}
//Timing.tick("Done with dependency phase.");
bool bothPassed = false;
if (op.doPCFG && op.doDep)
{
bothPassed = bparser.Parse(binaryTree.YieldHasWord());
}
//Timing.tick("Done with combination phase.");
long timeMil2 = Runtime.CurrentTimeMillis();
long elapsed = timeMil2 - timeMil1;
log.Info("Time: " + ((int)(elapsed / 100)) / 10.00 + " sec.");
//System.out.println("PCFG Best Parse:");
Tree tree2b = null;
Tree tree2 = null;
//System.out.println("Got full best parse...");
if (op.doPCFG)
{
tree2b = parser_1.GetBestParse();
tree2 = debinarizer.TransformTree(tree2b);
}
//System.out.println("Debinarized parse...");
//tree2.pennPrint();
//System.out.println("DepG Best Parse:");
Tree tree3 = null;
Tree tree3db = null;
if (op.doDep)
{
tree3 = dparser.GetBestParse();
// was: but wrong Tree tree3db = debinarizer.transformTree(tree2);
tree3db = debinarizer.TransformTree(tree3);
tree3.PennPrint(pw);
}
//tree.pennPrint();
//((Tree)binaryTrainTrees.get(tNum)).pennPrint();
//System.out.println("Combo Best Parse:");
Tree tree4 = null;
if (op.doPCFG && op.doDep)
{
try
{
tree4 = bparser.GetBestParse();
if (tree4 == null)
{
tree4 = tree2b;
}
}
catch (ArgumentNullException)
{
log.Info("Blocked, using PCFG parse!");
tree4 = tree2b;
}
}
if (op.doPCFG && !bothPassed)
{
tree4 = tree2b;
}
//tree4.pennPrint();
if (op.doDep)
{
depDE.Evaluate(tree3, binaryTree, pw);
depTE.Evaluate(tree3db, tree_2, pw);
}
ITreeTransformer tc = op.tlpParams.Collinizer();
ITreeTransformer tcEvalb = op.tlpParams.CollinizerEvalb();
if (op.doPCFG)
{
// System.out.println("XXXX Best PCFG was: ");
// tree2.pennPrint();
// System.out.println("XXXX Transformed best PCFG is: ");
// tc.transformTree(tree2).pennPrint();
//System.out.println("True Best Parse:");
//tree.pennPrint();
//tc.transformTree(tree).pennPrint();
pcfgPE.Evaluate(tc.TransformTree(tree2), tc.TransformTree(tree_2), pw);
pcfgCB.Evaluate(tc.TransformTree(tree2), tc.TransformTree(tree_2), pw);
Tree tree4b = null;
if (op.doDep)
{
comboDE.Evaluate((bothPassed ? tree4 : tree3), binaryTree, pw);
tree4b = tree4;
tree4 = debinarizer.TransformTree(tree4);
if (op.nodePrune)
{
NodePruner np = new NodePruner(parser_1, debinarizer);
tree4 = np.Prune(tree4);
}
//tree4.pennPrint();
comboPE.Evaluate(tc.TransformTree(tree4), tc.TransformTree(tree_2), pw);
}
//pcfgTE.evaluate(tree2, tree);
pcfgTE.Evaluate(tcEvalb.TransformTree(tree2), tcEvalb.TransformTree(tree_2), pw);
pcfgTEnoPunct.Evaluate(tc.TransformTree(tree2), tc.TransformTree(tree_2), pw);
if (op.doDep)
{
comboTE.Evaluate(tcEvalb.TransformTree(tree4), tcEvalb.TransformTree(tree_2), pw);
comboTEnoPunct.Evaluate(tc.TransformTree(tree4), tc.TransformTree(tree_2), pw);
}
System.Console.Out.WriteLine("PCFG only: " + parser_1.ScoreBinarizedTree(tree2b, 0));
//tc.transformTree(tree2).pennPrint();
tree2.PennPrint(pw);
if (op.doDep)
{
System.Console.Out.WriteLine("Combo: " + parser_1.ScoreBinarizedTree(tree4b, 0));
// tc.transformTree(tree4).pennPrint(pw);
tree4.PennPrint(pw);
}
System.Console.Out.WriteLine("Correct:" + parser_1.ScoreBinarizedTree(binaryTree, 0));
/*
* if (parser.scoreBinarizedTree(tree2b,true) < parser.scoreBinarizedTree(binaryTree,true)) {
* System.out.println("SCORE INVERSION");
* parser.validateBinarizedTree(binaryTree,0);
* }
*/
tree_2.PennPrint(pw);
}
// end if doPCFG
if (op.testOptions.evalb)
{
if (op.doPCFG && op.doDep)
{
EvalbFormatWriter.WriteEVALBline(tcEvalb.TransformTree(tree_2), tcEvalb.TransformTree(tree4));
}
else
{
if (op.doPCFG)
{
EvalbFormatWriter.WriteEVALBline(tcEvalb.TransformTree(tree_2), tcEvalb.TransformTree(tree2));
}
else
{
if (op.doDep)
{
EvalbFormatWriter.WriteEVALBline(tcEvalb.TransformTree(tree_2), tcEvalb.TransformTree(tree3db));
}
}
}
}
}
// end for each tree in test treebank
if (op.testOptions.evalb)
{
EvalbFormatWriter.CloseEVALBfiles();
}
// op.testOptions.display();
if (op.doPCFG)
{
pcfgPE.Display(false, pw);
System.Console.Out.WriteLine("Grammar size: " + stateIndex.Size());
pcfgCB.Display(false, pw);
if (op.doDep)
{
comboPE.Display(false, pw);
}
pcfgTE.Display(false, pw);
pcfgTEnoPunct.Display(false, pw);
if (op.doDep)
{
comboTE.Display(false, pw);
comboTEnoPunct.Display(false, pw);
}
}
if (op.doDep)
{
depTE.Display(false, pw);
depDE.Display(false, pw);
}
if (op.doPCFG && op.doDep)
{
comboDE.Display(false, pw);
}
}