private void Write(Tree t, PrintWriter pw) { if (this.taggedOutput) { pw.Println(ATBTreeUtils.TaggedStringFromTree(t, this._enclosing.removeEscapeTokens, this._enclosing.wordTagDelim)); } else { t.PennPrint(pw); } }
public override void VisitTree(Tree t) { if (t == null || t.Value().Equals("X")) { return; } t = t.Prune(this.nullFilter, new LabeledScoredTreeFactory()); //Do *not* strip traces here. The ArabicTreeReader will do that if needed foreach (Tree node in t) { if (node.IsPreTerminal()) { this.ProcessPreterminal(node); } } this.treesVisited++; string flatString = (this.makeFlatFile) ? ATBTreeUtils.FlattenTree(t) : null; //Do the decimation if (this.treesVisited % 9 == 0) { this.Write(t, this.outFiles[this.devExtension]); if (this.makeFlatFile) { this.outFiles[this.devExtension + this.flatExtension].Println(flatString); } } else { if (this.treesVisited % 10 == 0) { this.Write(t, this.outFiles[this.testExtension]); if (this.makeFlatFile) { this.outFiles[this.testExtension + this.flatExtension].Println(flatString); } } else { this.Write(t, this.outFiles[this.trainExtension]); if (this.makeFlatFile) { this.outFiles[this.trainExtension + this.flatExtension].Println(flatString); } } } }
public virtual void VisitTree(Tree t) { // Filter out XBar trees if (t == null || t.Value().Equals("X")) { return; } if (t.Yield().Count > this._enclosing.maxLen) { return; } // Strip out traces and pronoun deletion markers, t = t.Prune(this.nullFilter, this.tf); t = this.ArabicAoverAFilter(t); // Visit nodes with a custom visitor if (this._enclosing.customTreeVisitor != null) { this._enclosing.customTreeVisitor.VisitTree(t); } // Process each node in the tree foreach (Tree node in t) { if (node.IsPreTerminal()) { this.ProcessPreterminal(node); } if (this._enclosing.removeDashTags && !node.IsLeaf()) { node.SetValue(this.tlp.BasicCategory(node.Value())); } } // Add a ROOT node if necessary if (this._enclosing.addRoot && t.Value() != null && !t.Value().Equals("ROOT")) { t = this.tf.NewTreeNode("ROOT", Collections.SingletonList(t)); } // Output the trees to file this.outfile.Println(t.ToString()); if (this.flatFile != null) { string flatString = (this._enclosing.removeEscapeTokens) ? ATBTreeUtils.UnEscape(ATBTreeUtils.FlattenTree(t)) : ATBTreeUtils.FlattenTree(t); this.flatFile.Println(flatString); } }
public override void VisitTree(Tree t) { if (t == null || t.Value().Equals("X")) { return; } t = t.Prune(this.nullFilter, new LabeledScoredTreeFactory()); foreach (Tree node in t) { if (node.IsPreTerminal()) { this.ProcessPreterminal(node); } } this.outfile.Println(ATBTreeUtils.TaggedStringFromTree(t, this._enclosing.removeEscapeTokens, this._enclosing.wordTagDelim)); if (this.flatFile != null) { this.flatFile.Println(ATBTreeUtils.FlattenTree(t)); } }
/// <summary>Escapes a word.</summary> /// <remarks>Escapes a word. This method will *not* map a word to the null string.</remarks> /// <returns>The escaped string</returns> private string EscapeString(string word) { string firstStage = StripAnnotationsAndClassing(word); string secondStage = ATBTreeUtils.Escape(firstStage); if (secondStage.IsEmpty()) { return(firstStage); } else { if (!firstStage.Equals(secondStage)) { return(secondStage); } } string thirdStage = lexMapper.Map(null, secondStage); if (thirdStage.IsEmpty()) { return(secondStage); } return(thirdStage); }
public override void Build() { foreach (File path in pathsToData) { treebank.LoadPath(path, treeFileExtension, false); } PrintWriter outfile = null; PrintWriter flatFile = null; try { outfile = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFileName), "UTF-8"))); flatFile = (makeFlatFile) ? new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(flatFileName), "UTF-8"))) : null; outputFileList.Add(outFileName); if (makeFlatFile) { outputFileList.Add(flatFileName); toStringBuffer.Append(" Made flat files\n"); } PreprocessMWEs(); IList <TregexPattern> badTrees = new List <TregexPattern>(); //These trees appear in the Candito training set //They are mangled by the TreeCorrector, so discard them ahead of time. badTrees.Add(TregexPattern.Compile("@SENT <: @PUNC")); badTrees.Add(TregexPattern.Compile("@SENT <1 @PUNC <2 @PUNC !<3 __")); //wsg2011: This filters out tree #552 in the Candito test set. We saved this tree for the //EMNLP2011 paper, but since it consists entirely of punctuation, it won't be evaluated anyway. //Since we aren't doing the split in this data set, just remove the tree. badTrees.Add(TregexPattern.Compile("@SENT <1 @PUNC <2 @PUNC <3 @PUNC <4 @PUNC !<5 __")); foreach (Tree t in treebank) { //Filter out bad trees bool skipTree = false; foreach (TregexPattern p in badTrees) { skipTree = p.Matcher(t).Find(); if (skipTree) { break; } } if (skipTree) { log.Info("Discarding tree: " + t.ToString()); continue; } // Filter out trees that aren't in this part of the split if (splitSet != null) { string canditoTreeID = GetCanditoTreeID(t); if (!splitSet.Contains(canditoTreeID)) { continue; } } if (customTreeVisitor != null) { customTreeVisitor.VisitTree(t); } // outfile.printf("%s\t%s%n",treeName,t.toString()); outfile.Println(t.ToString()); if (makeFlatFile) { string flatString = (removeEscapeTokens) ? ATBTreeUtils.UnEscape(ATBTreeUtils.FlattenTree(t)) : ATBTreeUtils.FlattenTree(t); flatFile.Println(flatString); } } } catch (UnsupportedEncodingException e) { System.Console.Error.Printf("%s: Filesystem does not support UTF-8 output%n", this.GetType().FullName); Sharpen.Runtime.PrintStackTrace(e); } catch (FileNotFoundException) { System.Console.Error.Printf("%s: Could not open %s for writing%n", this.GetType().FullName, outFileName); } catch (TregexParseException e) { System.Console.Error.Printf("%s: Could not compile Tregex expressions%n", this.GetType().FullName); Sharpen.Runtime.PrintStackTrace(e); } finally { if (outfile != null) { outfile.Close(); } if (flatFile != null) { flatFile.Close(); } } }