public static void RemoveKeys <E, F>(TwoDimensionalCounter <E, F> counter, ICollection <E> removeKeysCollection) { foreach (E key in removeKeysCollection) { counter.Remove(key); } }
private void LoadMWMap(string filename) { mwCounter = new TwoDimensionalCounter <string, string>(); try { BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(filename)), "UTF-8")); int nLines = 0; for (string line; (line = br.ReadLine()) != null; nLines++) { string[] toks = line.Split("\t"); System.Diagnostics.Debug.Assert(toks.Length == 3); mwCounter.SetCount(toks[0].Trim(), toks[1].Trim(), double.Parse(toks[2].Trim())); } br.Close(); System.Console.Error.Printf("%s: Loaded %d lines from %s into MWE counter%n", this.GetType().FullName, nLines, filename); } catch (UnsupportedEncodingException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (FileNotFoundException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (IOException e) { Sharpen.Runtime.PrintStackTrace(e); } }
/// <summary> /// Attempt to infer the part of speech of the given preterminal node, which /// was created during the expansion of a multi-word token. /// </summary> private static string InferPOS(Tree t, Tree parent, TwoDimensionalCounter <string, string> unigramTagger) { string word = t.FirstChild().Value(); string containingPhraseStr = GetContainingPhrase(t, parent); // Overrides: let the manual POS model handle a few special cases first string overrideTag = MultiWordPreprocessor.ManualUWModel.GetOverrideTag(word, containingPhraseStr); if (overrideTag != null) { return(overrideTag); } ICollection <string> unigramTaggerKeys = unigramTagger.FirstKeySet(); // Try treating this word as a verb and stripping any clitic // pronouns. If the stripped version exists in the unigram // tagger, then stick with the verb hypothesis SpanishVerbStripper.StrippedVerb strippedVerb = verbStripper.SeparatePronouns(word); if (strippedVerb != null && unigramTaggerKeys.Contains(strippedVerb.GetStem())) { string pos = Counters.Argmax(unigramTagger.GetCounter(strippedVerb.GetStem())); if (pos.StartsWith("v")) { return(pos); } } if (unigramTagger.FirstKeySet().Contains(word)) { return(Counters.Argmax(unigramTagger.GetCounter(word), new MultiWordPreprocessor.POSTieBreaker())); } return(MultiWordPreprocessor.ManualUWModel.GetTag(word, containingPhraseStr)); }
private double NumNonRedundantPatterns(TwoDimensionalCounter <CandidatePhrase, E> terms, CandidatePhrase w) { object[] pats = Sharpen.Collections.ToArray(terms.GetCounter(w).KeySet()); int numPat = 0; for (int i = 0; i < pats.Length; i++) { //String pati = constVars.getPatternIndex().get(pats[i]).toString(); string pati = pats[i].ToString(); bool contains = false; for (int j = i + 1; j < pats.Length; j++) { //String patj = constVars.getPatternIndex().get(pats[j]).toString(); string patj = pats[j].ToString(); if (patj.Contains(pati) || pati.Contains(patj)) { contains = true; break; } } if (!contains) { numPat++; } } return(numPat); }
/// <exception cref="System.IO.IOException"/> public static void Main(string[] args) { if (args.Length < 1) { log.Info(usage); System.Environment.Exit(1); } Properties options = StringUtils.ArgsToProperties(args, argOptionDefs); string outputPath = options.GetProperty("o"); if (outputPath == null) { throw new ArgumentException("-o argument (output path for built tagger) is required"); } string[] remainingArgs = options.GetProperty(string.Empty).Split(" "); IList <File> fileList = new List <File>(); foreach (string arg in remainingArgs) { fileList.Add(new File(arg)); } Edu.Stanford.Nlp.International.Spanish.Pipeline.AnCoraPOSStats stats = new Edu.Stanford.Nlp.International.Spanish.Pipeline.AnCoraPOSStats(fileList, outputPath); stats.Process(); ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(outputPath)); TwoDimensionalCounter <string, string> tagger = stats.GetUnigramTagger(); oos.WriteObject(tagger); System.Console.Out.Printf("Wrote tagger to %s%n", outputPath); }
public static void TraverseAndFix(Tree t, Tree parent, TwoDimensionalCounter <string, string> unigramTagger, bool retainNER) { if (t.IsPreTerminal()) { if (t.Value().Equals(SpanishTreeNormalizer.MwTag)) { nMissingPOS++; string pos = InferPOS(t, parent, unigramTagger); if (pos != null) { t.SetValue(pos); nFixedPOS++; } } return; } foreach (Tree kid in t.Children()) { TraverseAndFix(kid, t, unigramTagger, retainNER); } // Post-order visit if (t.Value().StartsWith(SpanishTreeNormalizer.MwPhraseTag)) { nMissingPhrasal++; string phrasalCat = InferPhrasalCategory(t, retainNER); if (phrasalCat != null) { t.SetValue(phrasalCat); nFixedPhrasal++; } } }
/// <summary>Processes a single file containing AnCora XML trees.</summary> /// <remarks> /// Processes a single file containing AnCora XML trees. Returns MWE statistics for the trees in /// the file and the actual parsed trees. /// </remarks> private static Pair <TwoDimensionalCounter <string, string>, IList <Tree> > ProcessTreeFile(File file, SpanishXMLTreeReaderFactory trf, string encoding) { TwoDimensionalCounter <string, string> tagger = new TwoDimensionalCounter <string, string>(); try { Reader @in = new BufferedReader(new InputStreamReader(new FileInputStream(file), encoding)); ITreeReader tr = trf.NewTreeReader(file.GetPath(), @in); IList <Tree> trees = new List <Tree>(); Tree t; Tree splitPoint; while ((t = tr.ReadTree()) != null) { do { // We may need to split the current tree into multiple parts. // (If not, a call to `split` with a `null` split-point is a // no-op splitPoint = FindSplitPoint(t); Pair <Tree, Tree> split = Split(t, splitPoint); Tree toAdd = split.First(); t = split.Second(); trees.Add(toAdd); UpdateTagger(tagger, toAdd); }while (splitPoint != null); } tr.Close(); return(new Pair <TwoDimensionalCounter <string, string>, IList <Tree> >(tagger, trees)); } catch (IOException e) { Sharpen.Runtime.PrintStackTrace(e); return(null); } }
private static void ResolveDummyTags(File treeFile, TwoDimensionalCounter <string, string> pretermLabel, TwoDimensionalCounter <string, string> unigramTagger) { try { BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(treeFile), "UTF-8")); ITreeReaderFactory trf = new FrenchTreeReaderFactory(); ITreeReader tr = trf.NewTreeReader(br); PrintWriter pw = new PrintWriter(new TextWriter(new FileOutputStream(new File(treeFile + ".fixed")), false, "UTF-8")); int nTrees = 0; for (Tree t; (t = tr.ReadTree()) != null; nTrees++) { TraverseAndFix(t, pretermLabel, unigramTagger); pw.Println(t.ToString()); } pw.Close(); tr.Close(); System.Console.Out.WriteLine("Processed " + nTrees + " trees"); } catch (UnsupportedEncodingException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (FileNotFoundException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (IOException e) { Sharpen.Runtime.PrintStackTrace(e); } }
public ScorePatternsF1(ConstantsAndVariables constVars, GetPatternsFromDataMultiClass.PatternScoring patternScoring, string label, ICollection <CandidatePhrase> allCandidatePhrases, TwoDimensionalCounter <E, CandidatePhrase> patternsandWords4Label , TwoDimensionalCounter <E, CandidatePhrase> negPatternsandWords4Label, TwoDimensionalCounter <E, CandidatePhrase> unLabeledPatternsandWords4Label, Properties props, ICounter <CandidatePhrase> p0Set, E p0) : base(constVars, patternScoring, label, allCandidatePhrases, patternsandWords4Label, negPatternsandWords4Label, unLabeledPatternsandWords4Label, props) { this.p0 = p0; this.p0Set = p0Set; }
public ScorePatternsRatioModifiedFreq(ConstantsAndVariables constVars, GetPatternsFromDataMultiClass.PatternScoring patternScoring, string label, ICollection <CandidatePhrase> allCandidatePhrases, TwoDimensionalCounter <E, CandidatePhrase> patternsandWords4Label , TwoDimensionalCounter <E, CandidatePhrase> negPatternsandWords4Label, TwoDimensionalCounter <E, CandidatePhrase> unLabeledPatternsandWords4Label, TwoDimensionalCounter <CandidatePhrase, ConstantsAndVariables.ScorePhraseMeasures> phInPatScores , ScorePhrases scorePhrases, Properties props) : base(constVars, patternScoring, label, allCandidatePhrases, patternsandWords4Label, negPatternsandWords4Label, unLabeledPatternsandWords4Label, props) { this.phInPatScores = phInPatScores; this.scorePhrases = scorePhrases; }
protected virtual void SetUp() { c = new TwoDimensionalCounter <string, string>(); c.SetCount("a", "a", 1.0); c.SetCount("a", "b", 2.0); c.SetCount("a", "c", 3.0); c.SetCount("b", "a", 4.0); c.SetCount("b", "b", 5.0); c.SetCount("c", "a", 6.0); }
/// <param name="args"/> public static void Main(string[] args) { if (args.Length != 1) { System.Console.Error.Printf("Usage: java %s file%n", typeof(MWEPreprocessor).FullName); System.Environment.Exit(-1); } File treeFile = new File(args[0]); TwoDimensionalCounter <string, string> labelTerm = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> termLabel = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> labelPreterm = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> pretermLabel = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> unigramTagger = new TwoDimensionalCounter <string, string>(); try { BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(treeFile), "UTF-8")); ITreeReaderFactory trf = new FrenchTreeReaderFactory(); ITreeReader tr = trf.NewTreeReader(br); for (Tree t; (t = tr.ReadTree()) != null;) { CountMWEStatistics(t, unigramTagger, labelPreterm, pretermLabel, labelTerm, termLabel); } tr.Close(); //Closes the underlying reader System.Console.Out.WriteLine("Generating {MWE Type -> Terminal}"); PrintCounter(labelTerm, "label_term.csv"); System.Console.Out.WriteLine("Generating {Terminal -> MWE Type}"); PrintCounter(termLabel, "term_label.csv"); System.Console.Out.WriteLine("Generating {MWE Type -> POS sequence}"); PrintCounter(labelPreterm, "label_pos.csv"); System.Console.Out.WriteLine("Generating {POS sequence -> MWE Type}"); PrintCounter(pretermLabel, "pos_label.csv"); System.Console.Out.WriteLine("Resolving DUMMY tags"); ResolveDummyTags(treeFile, pretermLabel, unigramTagger); System.Console.Out.WriteLine("#Unknown Word Types: " + MWEPreprocessor.ManualUWModel.nUnknownWordTypes); System.Console.Out.WriteLine("#Missing POS: " + nMissingPOS); System.Console.Out.WriteLine("#Missing Phrasal: " + nMissingPhrasal); System.Console.Out.WriteLine("Done!"); } catch (UnsupportedEncodingException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (FileNotFoundException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (IOException e) { Sharpen.Runtime.PrintStackTrace(e); } }
private static void UpdateTagger(TwoDimensionalCounter <string, string> tagger, Tree t) { IList <CoreLabel> yield = t.TaggedLabeledYield(); foreach (CoreLabel label in yield) { if (label.Tag().Equals(SpanishTreeNormalizer.MwTag)) { continue; } tagger.IncrementCount(label.Word(), label.Tag()); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.TypeLoadException"/> public AnCoraProcessor(IList <File> inputFiles, Properties options) { this.inputFiles = inputFiles; this.options = options; if (options.Contains("unigramTagger")) { ObjectInputStream ois = new ObjectInputStream(new FileInputStream(options.GetProperty("unigramTagger"))); unigramTagger = (TwoDimensionalCounter <string, string>)ois.ReadObject(); } else { unigramTagger = new TwoDimensionalCounter <string, string>(); } }
/// <param name="args"/> public static void Main(string[] args) { Properties options = StringUtils.ArgsToProperties(args, argOptionDefs); if (!options.Contains(string.Empty) || options.Contains("help")) { log.Info(Usage()); return; } bool retainNER = PropertiesUtils.GetBool(options, "ner", false); bool normalize = PropertiesUtils.GetBool(options, "normalize", true); File treeFile = new File(options.GetProperty(string.Empty)); TwoDimensionalCounter <string, string> labelTerm = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> termLabel = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> labelPreterm = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> pretermLabel = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> unigramTagger = new TwoDimensionalCounter <string, string>(); try { BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(treeFile), "UTF-8")); ITreeReaderFactory trf = new SpanishTreeReaderFactory(); ITreeReader tr = trf.NewTreeReader(br); for (Tree t; (t = tr.ReadTree()) != null;) { UpdateTagger(unigramTagger, t); } tr.Close(); //Closes the underlying reader System.Console.Out.WriteLine("Resolving DUMMY tags"); ResolveDummyTags(treeFile, unigramTagger, retainNER, normalize ? new SpanishTreeNormalizer(true, false, false) : null); System.Console.Out.WriteLine("#Unknown Word Types: " + MultiWordPreprocessor.ManualUWModel.nUnknownWordTypes); System.Console.Out.WriteLine(string.Format("#Missing POS: %d (fixed: %d, %.2f%%)", nMissingPOS, nFixedPOS, (double)nFixedPOS / nMissingPOS * 100)); System.Console.Out.WriteLine(string.Format("#Missing Phrasal: %d (fixed: %d, %.2f%%)", nMissingPhrasal, nFixedPhrasal, (double)nFixedPhrasal / nMissingPhrasal * 100)); System.Console.Out.WriteLine("Done!"); } catch (UnsupportedEncodingException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (FileNotFoundException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (IOException e) { Sharpen.Runtime.PrintStackTrace(e); } }
public static void UpdateTagger(TwoDimensionalCounter <string, string> tagger, Tree t) { IList <CoreLabel> yield = t.TaggedLabeledYield(); foreach (CoreLabel cl in yield) { if (ResolveDummyTags && cl.Tag().Equals(FrenchXMLTreeReader.MissingPos)) { continue; } else { tagger.IncrementCount(cl.Word(), cl.Tag()); } } }
internal static void PreprocessMWEs(IDictionary <string, Tree> treeMap) { TwoDimensionalCounter <string, string> labelTerm = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> termLabel = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> labelPreterm = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> pretermLabel = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> unigramTagger = new TwoDimensionalCounter <string, string>(); foreach (Tree t in treeMap.Values) { MWEPreprocessor.CountMWEStatistics(t, unigramTagger, labelPreterm, pretermLabel, labelTerm, termLabel); } foreach (Tree t_1 in treeMap.Values) { MWEPreprocessor.TraverseAndFix(t_1, pretermLabel, unigramTagger); } }
/// <summary>Corrects MWE annotations that lack internal POS labels.</summary> private void PreprocessMWEs() { TwoDimensionalCounter <string, string> labelTerm = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> termLabel = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> labelPreterm = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> pretermLabel = new TwoDimensionalCounter <string, string>(); TwoDimensionalCounter <string, string> unigramTagger = new TwoDimensionalCounter <string, string>(); foreach (Tree t in treebank) { MWEPreprocessor.CountMWEStatistics(t, unigramTagger, labelPreterm, pretermLabel, labelTerm, termLabel); } foreach (Tree t_1 in treebank) { MWEPreprocessor.TraverseAndFix(t_1, pretermLabel, unigramTagger); } }
public ScorePatterns(ConstantsAndVariables constVars, GetPatternsFromDataMultiClass.PatternScoring patternScoring, string label, ICollection <CandidatePhrase> allCandidatePhrases, TwoDimensionalCounter <E, CandidatePhrase> patternsandWords4Label , TwoDimensionalCounter <E, CandidatePhrase> negPatternsandWords4Label, TwoDimensionalCounter <E, CandidatePhrase> unLabeledPatternsandWords4Label, Properties props) { // protected TwoDimensionalCounter<SurfacePattern, String> // posnegPatternsandWords4Label = new TwoDimensionalCounter<SurfacePattern, // String>(); //protected TwoDimensionalCounter<E, String> negandUnLabeledPatternsandWords4Label = new TwoDimensionalCounter<E, String>(); //protected TwoDimensionalCounter<E, String> allPatternsandWords4Label = new TwoDimensionalCounter<E, String>(); this.constVars = constVars; this.patternScoring = patternScoring; this.label = label; this.allCandidatePhrases = allCandidatePhrases; this.patternsandWords4Label = patternsandWords4Label; this.negPatternsandWords4Label = negPatternsandWords4Label; this.unLabeledPatternsandWords4Label = unLabeledPatternsandWords4Label; this.props = props; }
private static void CountTaggings(Treebank tb, PrintWriter pw) { TwoDimensionalCounter <string, string> wtc = new TwoDimensionalCounter <string, string>(); tb.Apply(null); foreach (string key in wtc.FirstKeySet()) { pw.Print(key); pw.Print('\t'); ICounter <string> ctr = wtc.GetCounter(key); foreach (string k2 in ctr.KeySet()) { pw.Print(k2 + '\t' + ctr.GetCount(k2) + '\t'); } pw.Println(); } }
public virtual void ApplyPats(ICounter <E> patterns, string label, TwoDimensionalCounter <CandidatePhrase, E> wordsandLemmaPatExtracted, CollectionValuedMap <E, Triple <string, int, int> > matchedTokensByPat, ICollection <CandidatePhrase> alreadyLabeledWords ) { // Counter<E> patternsLearnedThisIterConsistsOnlyGeneralized = new ClassicCounter<E>(); // Counter<E> patternsLearnedThisIterRest = new ClassicCounter<E>(); // Set<String> specialWords = constVars.invertedIndex.getSpecialWordsList(); foreach (KeyValuePair <string, Env> en in constVars.env) { en.Value.GetVariables().PutAll(ConstantsAndVariables.globalEnv.GetVariables()); } IDictionary <E, IDictionary <string, DataInstance> > sentencesForPatterns = GetSentences(constVars.invertedIndex.QueryIndex(patterns.KeySet())); foreach (KeyValuePair <E, IDictionary <string, DataInstance> > en_1 in sentencesForPatterns) { RunParallelApplyPats(en_1.Value, label, en_1.Key, wordsandLemmaPatExtracted, matchedTokensByPat, alreadyLabeledWords); } Redwood.Log(Redwood.Dbg, "# words/lemma and pattern pairs are " + wordsandLemmaPatExtracted.Size()); }
private static void ResolveDummyTags(File treeFile, TwoDimensionalCounter <string, string> unigramTagger, bool retainNER, TreeNormalizer tn) { ITreeFactory tf = new LabeledScoredTreeFactory(); MultiWordTreeExpander expander = new MultiWordTreeExpander(); try { BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(treeFile), "UTF-8")); ITreeReaderFactory trf = new SpanishTreeReaderFactory(); ITreeReader tr = trf.NewTreeReader(br); PrintWriter pw = new PrintWriter(new TextWriter(new FileOutputStream(new File(treeFile + ".fixed")), false, "UTF-8")); int nTrees = 0; for (Tree t; (t = tr.ReadTree()) != null; nTrees++) { TraverseAndFix(t, null, unigramTagger, retainNER); // Now "decompress" further the expanded trees formed by // multiword token splitting t = expander.ExpandPhrases(t, tn, tf); if (tn != null) { t = tn.NormalizeWholeTree(t, tf); } pw.Println(t.ToString()); } pw.Close(); tr.Close(); System.Console.Out.WriteLine("Processed " + nTrees + " trees"); } catch (UnsupportedEncodingException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (FileNotFoundException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (IOException e) { Sharpen.Runtime.PrintStackTrace(e); } }
public static void TraverseAndFix(Tree t, TwoDimensionalCounter <string, string> pretermLabel, TwoDimensionalCounter <string, string> unigramTagger) { if (t.IsPreTerminal()) { if (t.Value().Equals(FrenchXMLTreeReader.MissingPos)) { nMissingPOS++; string word = t.FirstChild().Value(); string tag = (unigramTagger.FirstKeySet().Contains(word)) ? Counters.Argmax(unigramTagger.GetCounter(word)) : MWEPreprocessor.ManualUWModel.GetTag(word); t.SetValue(tag); } return; } foreach (Tree kid in t.Children()) { TraverseAndFix(kid, pretermLabel, unigramTagger); } //Post-order visit if (t.Value().Equals(FrenchXMLTreeReader.MissingPhrasal)) { nMissingPhrasal++; StringBuilder sb = new StringBuilder(); foreach (Tree kid_1 in t.Children()) { sb.Append(kid_1.Value()).Append(" "); } string posSequence = sb.ToString().Trim(); if (pretermLabel.FirstKeySet().Contains(posSequence)) { string phrasalCat = Counters.Argmax(pretermLabel.GetCounter(posSequence)); t.SetValue(phrasalCat); } else { System.Console.Out.WriteLine("No phrasal cat for: " + posSequence); } } }
public static void CountMWEStatistics(Tree t, TwoDimensionalCounter <string, string> unigramTagger, TwoDimensionalCounter <string, string> labelPreterm, TwoDimensionalCounter <string, string> pretermLabel, TwoDimensionalCounter <string, string> labelTerm, TwoDimensionalCounter <string, string> termLabel) { UpdateTagger(unigramTagger, t); //Count MWE statistics TregexMatcher m = pMWE.Matcher(t); while (m.FindNextMatchingNode()) { Tree match = m.GetMatch(); string label = match.Value(); if (ResolveDummyTags && label.Equals(FrenchXMLTreeReader.MissingPhrasal)) { continue; } string preterm = SentenceUtils.ListToString(match.PreTerminalYield()); string term = SentenceUtils.ListToString(match.Yield()); labelPreterm.IncrementCount(label, preterm); pretermLabel.IncrementCount(preterm, label); labelTerm.IncrementCount(label, term); termLabel.IncrementCount(term, label); } }
public static void PrintCounter(TwoDimensionalCounter <string, string> cnt, string fname) { try { PrintWriter pw = new PrintWriter(new TextWriter(new FileOutputStream(new File(fname)), false, "UTF-8")); foreach (string key in cnt.FirstKeySet()) { foreach (string val in cnt.GetCounter(key).KeySet()) { pw.Printf("%s\t%s\t%d%n", key, val, (int)cnt.GetCount(key, val)); } } pw.Close(); } catch (UnsupportedEncodingException e) { Sharpen.Runtime.PrintStackTrace(e); } catch (FileNotFoundException e) { Sharpen.Runtime.PrintStackTrace(e); } }
private void RunParallelApplyPats(IDictionary <string, DataInstance> sents, string label, E pattern, TwoDimensionalCounter <CandidatePhrase, E> wordsandLemmaPatExtracted, CollectionValuedMap <E, Triple <string, int, int> > matchedTokensByPat, ICollection <CandidatePhrase> alreadyLabeledWords) { Redwood.Log(Redwood.Dbg, "Applying pattern " + pattern + " to a total of " + sents.Count + " sentences "); IList <string> notAllowedClasses = new List <string>(); IList <string> sentids = CollectionUtils.ToList(sents.Keys); if (constVars.doNotExtractPhraseAnyWordLabeledOtherClass) { foreach (string l in constVars.GetAnswerClass().Keys) { if (!l.Equals(label)) { notAllowedClasses.Add(l); } } notAllowedClasses.Add("OTHERSEM"); } IDictionary <TokenSequencePattern, E> surfacePatternsLearnedThisIterConverted = null; IDictionary <SemgrexPattern, E> depPatternsLearnedThisIterConverted = null; if (constVars.patternType.Equals(PatternFactory.PatternType.Surface)) { surfacePatternsLearnedThisIterConverted = new Dictionary <TokenSequencePattern, E>(); string patternStr = null; try { patternStr = pattern.ToString(notAllowedClasses); TokenSequencePattern pat = ((TokenSequencePattern)TokenSequencePattern.Compile(constVars.env[label], patternStr)); surfacePatternsLearnedThisIterConverted[pat] = pattern; } catch (Exception e) { log.Info("Error applying pattern " + patternStr + ". Probably an ill formed pattern (can be because of special symbols in label names). Contact the software developer."); throw; } } else { if (constVars.patternType.Equals(PatternFactory.PatternType.Dep)) { depPatternsLearnedThisIterConverted = new Dictionary <SemgrexPattern, E>(); SemgrexPattern pat = SemgrexPattern.Compile(pattern.ToString(notAllowedClasses), new Env(constVars.env[label].GetVariables())); depPatternsLearnedThisIterConverted[pat] = pattern; } else { throw new NotSupportedException(); } } //Apply the patterns and extract candidate phrases int num; int numThreads = constVars.numThreads; //If number of sentences is less, do not create so many threads if (sents.Count < 50) { numThreads = 1; } if (numThreads == 1) { num = sents.Count; } else { num = sents.Count / (numThreads - 1); } IExecutorService executor = Executors.NewFixedThreadPool(constVars.numThreads); IList <IFuture <Triple <TwoDimensionalCounter <CandidatePhrase, E>, CollectionValuedMap <E, Triple <string, int, int> >, ICollection <CandidatePhrase> > > > list = new List <IFuture <Triple <TwoDimensionalCounter <CandidatePhrase, E>, CollectionValuedMap <E , Triple <string, int, int> >, ICollection <CandidatePhrase> > > >(); for (int i = 0; i < numThreads; i++) { ICallable <Triple <TwoDimensionalCounter <CandidatePhrase, E>, CollectionValuedMap <E, Triple <string, int, int> >, ICollection <CandidatePhrase> > > task = null; if (pattern.type.Equals(PatternFactory.PatternType.Surface)) { //Redwood.log(Redwood.DBG, "Applying pats: assigning sentences " + i*num + " to " +Math.min(sentids.size(), (i + 1) * num) + " to thread " + (i+1)); task = new ApplyPatterns(sents, num == sents.Count ? sentids : sentids.SubList(i * num, Math.Min(sentids.Count, (i + 1) * num)), surfacePatternsLearnedThisIterConverted, label, constVars.removeStopWordsFromSelectedPhrases, constVars.removePhrasesWithStopWords , constVars); } else { task = new ApplyDepPatterns(sents, num == sents.Count ? sentids : sentids.SubList(i * num, Math.Min(sentids.Count, (i + 1) * num)), depPatternsLearnedThisIterConverted, label, constVars.removeStopWordsFromSelectedPhrases, constVars.removePhrasesWithStopWords , constVars); } IFuture <Triple <TwoDimensionalCounter <CandidatePhrase, E>, CollectionValuedMap <E, Triple <string, int, int> >, ICollection <CandidatePhrase> > > submit = executor.Submit(task); list.Add(submit); } // Now retrieve the result foreach (IFuture <Triple <TwoDimensionalCounter <CandidatePhrase, E>, CollectionValuedMap <E, Triple <string, int, int> >, ICollection <CandidatePhrase> > > future in list) { try { Triple <TwoDimensionalCounter <CandidatePhrase, E>, CollectionValuedMap <E, Triple <string, int, int> >, ICollection <CandidatePhrase> > result = future.Get(); Redwood.Log(ConstantsAndVariables.extremedebug, "Pattern " + pattern + " extracted phrases " + result.First()); wordsandLemmaPatExtracted.AddAll(result.First()); matchedTokensByPat.AddAll(result.Second()); Sharpen.Collections.AddAll(alreadyLabeledWords, result.Third()); } catch (Exception e) { executor.ShutdownNow(); throw new Exception(e); } } executor.Shutdown(); }
/// <exception cref="System.Exception"/> public virtual Triple <TwoDimensionalCounter <CandidatePhrase, E>, CollectionValuedMap <E, Triple <string, int, int> >, ICollection <CandidatePhrase> > Call() { // CollectionValuedMap<String, Integer> tokensMatchedPattern = new // CollectionValuedMap<String, Integer>(); try { ICollection <CandidatePhrase> alreadyLabeledPhrases = new HashSet <CandidatePhrase>(); TwoDimensionalCounter <CandidatePhrase, E> allFreq = new TwoDimensionalCounter <CandidatePhrase, E>(); CollectionValuedMap <E, Triple <string, int, int> > matchedTokensByPat = new CollectionValuedMap <E, Triple <string, int, int> >(); foreach (string sentid in sentids) { IList <CoreLabel> sent = sents[sentid].GetTokens(); foreach (KeyValuePair <TokenSequencePattern, E> pEn in patterns) { if (pEn.Key == null) { throw new Exception("why is the pattern " + pEn + " null?"); } TokenSequenceMatcher m = ((TokenSequenceMatcher)pEn.Key.GetMatcher(sent)); // //Setting this find type can save time in searching - greedy and reluctant quantifiers are not enforced // m.setFindType(SequenceMatcher.FindType.FIND_ALL); //Higher branch values makes the faster but uses more memory m.SetBranchLimit(5); while (m.Find()) { int s = m.Start("$term"); int e = m.End("$term"); System.Diagnostics.Debug.Assert(e - s <= PatternFactory.numWordsCompoundMapped[label], "How come the pattern " + pEn.Key + " is extracting phrases longer than numWordsCompound of " + PatternFactory.numWordsCompoundMapped[label] + " for label " + label); string phrase = string.Empty; string phraseLemma = string.Empty; bool useWordNotLabeled = false; bool doNotUse = false; //find if the neighboring words are labeled - if so - club them together if (constVars.clubNeighboringLabeledWords) { for (int i = s - 1; i >= 0; i--) { if (!sent[i].Get(constVars.GetAnswerClass()[label]).Equals(label)) { s = i + 1; break; } } for (int i_1 = e; i_1 < sent.Count; i_1++) { if (!sent[i_1].Get(constVars.GetAnswerClass()[label]).Equals(label)) { e = i_1; break; } } } //to make sure we discard phrases with stopwords in between, but include the ones in which stop words were removed at the ends if removeStopWordsFromSelectedPhrases is true bool[] addedindices = new bool[e - s]; // Arrays.fill(addedindices, false); // not needed as initialized false for (int i_2 = s; i_2 < e; i_2++) { CoreLabel l = sent[i_2]; l.Set(typeof(PatternsAnnotations.MatchedPattern), true); if (!l.ContainsKey(typeof(PatternsAnnotations.MatchedPatterns)) || l.Get(typeof(PatternsAnnotations.MatchedPatterns)) == null) { l.Set(typeof(PatternsAnnotations.MatchedPatterns), new HashSet <Pattern>()); } SurfacePattern pSur = (SurfacePattern)pEn.Value; System.Diagnostics.Debug.Assert(pSur != null, "Why is " + pEn.Value + " not present in the index?!"); System.Diagnostics.Debug.Assert(l.Get(typeof(PatternsAnnotations.MatchedPatterns)) != null, "How come MatchedPatterns class is null for the token. The classes in the key set are " + l.KeySet()); l.Get(typeof(PatternsAnnotations.MatchedPatterns)).Add(pSur); foreach (KeyValuePair <Type, object> ig in constVars.GetIgnoreWordswithClassesDuringSelection()[label]) { if (l.ContainsKey(ig.Key) && l.Get(ig.Key).Equals(ig.Value)) { doNotUse = true; } } bool containsStop = ContainsStopWord(l, constVars.GetCommonEngWords(), PatternFactory.ignoreWordRegex); if (removePhrasesWithStopWords && containsStop) { doNotUse = true; } else { if (!containsStop || !removeStopWordsFromSelectedPhrases) { if (label == null || l.Get(constVars.GetAnswerClass()[label]) == null || !l.Get(constVars.GetAnswerClass()[label]).Equals(label)) { useWordNotLabeled = true; } phrase += " " + l.Word(); phraseLemma += " " + l.Lemma(); addedindices[i_2 - s] = true; } } } for (int i_3 = 0; i_3 < addedindices.Length; i_3++) { if (i_3 > 0 && i_3 < addedindices.Length - 1 && addedindices[i_3 - 1] == true && addedindices[i_3] == false && addedindices[i_3 + 1] == true) { doNotUse = true; break; } } if (!doNotUse) { matchedTokensByPat.Add(pEn.Value, new Triple <string, int, int>(sentid, s, e - 1)); phrase = phrase.Trim(); if (!phrase.IsEmpty()) { phraseLemma = phraseLemma.Trim(); CandidatePhrase candPhrase = CandidatePhrase.CreateOrGet(phrase, phraseLemma); allFreq.IncrementCount(candPhrase, pEn.Value, 1.0); if (!useWordNotLabeled) { alreadyLabeledPhrases.Add(candPhrase); } } } } } } return(new Triple <TwoDimensionalCounter <CandidatePhrase, E>, CollectionValuedMap <E, Triple <string, int, int> >, ICollection <CandidatePhrase> >(allFreq, matchedTokensByPat, alreadyLabeledPhrases)); } catch (Exception e) { logger.Error(e); throw; } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.TypeLoadException"/> private ICounter <CandidatePhrase> LearnNewPhrasesPrivate(string label, PatternsForEachToken patternsForEachToken, ICounter <E> patternsLearnedThisIter, ICounter <E> allSelectedPatterns, ICollection <CandidatePhrase> alreadyIdentifiedWords, CollectionValuedMap <E, Triple <string, int, int> > matchedTokensByPat, ICounter <CandidatePhrase> scoreForAllWordsThisIteration, TwoDimensionalCounter <CandidatePhrase, E> terms, TwoDimensionalCounter <CandidatePhrase, E> wordsPatExtracted, TwoDimensionalCounter <E , CandidatePhrase> patternsAndWords4Label, string identifier, ICollection <CandidatePhrase> ignoreWords, bool computeProcDataFreq) { ICollection <CandidatePhrase> alreadyLabeledWords = new HashSet <CandidatePhrase>(); if (constVars.doNotApplyPatterns) { // if want to get the stats by the lossy way of just counting without // applying the patterns ConstantsAndVariables.DataSentsIterator sentsIter = new ConstantsAndVariables.DataSentsIterator(constVars.batchProcessSents); while (sentsIter.MoveNext()) { Pair <IDictionary <string, DataInstance>, File> sentsf = sentsIter.Current; this.StatsWithoutApplyingPatterns(sentsf.First(), patternsForEachToken, patternsLearnedThisIter, wordsPatExtracted); } } else { if (patternsLearnedThisIter.Size() > 0) { this.ApplyPats(patternsLearnedThisIter, label, wordsPatExtracted, matchedTokensByPat, alreadyLabeledWords); } } if (computeProcDataFreq) { if (!phraseScorer.wordFreqNorm.Equals(PhraseScorer.Normalization.None)) { Redwood.Log(Redwood.Dbg, "computing processed freq"); foreach (KeyValuePair <CandidatePhrase, double> fq in Data.rawFreq.EntrySet()) { double @in = fq.Value; if (phraseScorer.wordFreqNorm.Equals(PhraseScorer.Normalization.Sqrt)) { @in = Math.Sqrt(@in); } else { if (phraseScorer.wordFreqNorm.Equals(PhraseScorer.Normalization.Log)) { @in = 1 + Math.Log(@in); } else { throw new Exception("can't understand the normalization"); } } System.Diagnostics.Debug.Assert(!double.IsNaN(@in), "Why is processed freq nan when rawfreq is " + @in); Data.processedDataFreq.SetCount(fq.Key, @in); } } else { Data.processedDataFreq = Data.rawFreq; } } if (constVars.wordScoring.Equals(GetPatternsFromDataMultiClass.WordScoring.Weightednorm)) { foreach (CandidatePhrase en in wordsPatExtracted.FirstKeySet()) { if (!constVars.GetOtherSemanticClassesWords().Contains(en) && (en.GetPhraseLemma() == null || !constVars.GetOtherSemanticClassesWords().Contains(CandidatePhrase.CreateOrGet(en.GetPhraseLemma()))) && !alreadyLabeledWords.Contains(en)) { terms.AddAll(en, wordsPatExtracted.GetCounter(en)); } } RemoveKeys(terms, ConstantsAndVariables.GetStopWords()); ICounter <CandidatePhrase> phraseScores = phraseScorer.ScorePhrases(label, terms, wordsPatExtracted, allSelectedPatterns, alreadyIdentifiedWords, false); System.Console.Out.WriteLine("count for word U.S. is " + phraseScores.GetCount(CandidatePhrase.CreateOrGet("U.S."))); ICollection <CandidatePhrase> ignoreWordsAll; if (ignoreWords != null && !ignoreWords.IsEmpty()) { ignoreWordsAll = CollectionUtils.UnionAsSet(ignoreWords, constVars.GetOtherSemanticClassesWords()); } else { ignoreWordsAll = new HashSet <CandidatePhrase>(constVars.GetOtherSemanticClassesWords()); } Sharpen.Collections.AddAll(ignoreWordsAll, constVars.GetSeedLabelDictionary()[label]); Sharpen.Collections.AddAll(ignoreWordsAll, constVars.GetLearnedWords(label).KeySet()); System.Console.Out.WriteLine("ignoreWordsAll contains word U.S. is " + ignoreWordsAll.Contains(CandidatePhrase.CreateOrGet("U.S."))); ICounter <CandidatePhrase> finalwords = ChooseTopWords(phraseScores, terms, phraseScores, ignoreWordsAll, constVars.thresholdWordExtract); phraseScorer.PrintReasonForChoosing(finalwords); scoreForAllWordsThisIteration.Clear(); Counters.AddInPlace(scoreForAllWordsThisIteration, phraseScores); Redwood.Log(ConstantsAndVariables.minimaldebug, "\n\n## Selected Words for " + label + " : " + Counters.ToSortedString(finalwords, finalwords.Size(), "%1$s:%2$.2f", "\t")); if (constVars.goldEntities != null) { IDictionary <string, bool> goldEntities4Label = constVars.goldEntities[label]; if (goldEntities4Label != null) { StringBuilder s = new StringBuilder(); finalwords.KeySet().Stream().ForEach(null); Redwood.Log(ConstantsAndVariables.minimaldebug, "\n\n## Gold labels for selected words for label " + label + " : " + s.ToString()); } else { Redwood.Log(Redwood.Dbg, "No gold entities provided for label " + label); } } if (constVars.outDir != null && !constVars.outDir.IsEmpty()) { string outputdir = constVars.outDir + "/" + identifier + "/" + label; IOUtils.EnsureDir(new File(outputdir)); TwoDimensionalCounter <CandidatePhrase, CandidatePhrase> reasonForWords = new TwoDimensionalCounter <CandidatePhrase, CandidatePhrase>(); foreach (CandidatePhrase word in finalwords.KeySet()) { foreach (E l in wordsPatExtracted.GetCounter(word).KeySet()) { foreach (CandidatePhrase w2 in patternsAndWords4Label.GetCounter(l)) { reasonForWords.IncrementCount(word, w2); } } } Redwood.Log(ConstantsAndVariables.minimaldebug, "Saving output in " + outputdir); string filename = outputdir + "/words.json"; // the json object is an array corresponding to each iteration - of list // of objects, // each of which is a bean of entity and reasons IJsonArrayBuilder obj = Javax.Json.Json.CreateArrayBuilder(); if (writtenInJustification.Contains(label) && writtenInJustification[label]) { IJsonReader jsonReader = Javax.Json.Json.CreateReader(new BufferedInputStream(new FileInputStream(filename))); IJsonArray objarr = jsonReader.ReadArray(); foreach (IJsonValue o in objarr) { obj.Add(o); } jsonReader.Close(); } IJsonArrayBuilder objThisIter = Javax.Json.Json.CreateArrayBuilder(); foreach (CandidatePhrase w in reasonForWords.FirstKeySet()) { IJsonObjectBuilder objinner = Javax.Json.Json.CreateObjectBuilder(); IJsonArrayBuilder l = Javax.Json.Json.CreateArrayBuilder(); foreach (CandidatePhrase w2 in reasonForWords.GetCounter(w).KeySet()) { l.Add(w2.GetPhrase()); } IJsonArrayBuilder pats = Javax.Json.Json.CreateArrayBuilder(); foreach (E p in wordsPatExtracted.GetCounter(w)) { pats.Add(p.ToStringSimple()); } objinner.Add("reasonwords", l); objinner.Add("patterns", pats); objinner.Add("score", finalwords.GetCount(w)); objinner.Add("entity", w.GetPhrase()); objThisIter.Add(objinner.Build()); } obj.Add(objThisIter); // Redwood.log(ConstantsAndVariables.minimaldebug, channelNameLogger, // "Writing justification at " + filename); IOUtils.WriteStringToFile(StringUtils.Normalize(StringUtils.ToAscii(obj.Build().ToString())), filename, "ASCII"); writtenInJustification[label] = true; } if (constVars.justify) { Redwood.Log(Redwood.Dbg, "\nJustification for phrases:\n"); foreach (CandidatePhrase word in finalwords.KeySet()) { Redwood.Log(Redwood.Dbg, "Phrase " + word + " extracted because of patterns: \t" + Counters.ToSortedString(wordsPatExtracted.GetCounter(word), wordsPatExtracted.GetCounter(word).Size(), "%1$s:%2$f", "\n")); } } // if (usePatternResultAsLabel) // if (answerLabel != null) // labelWords(sents, commonEngWords, finalwords.keySet(), // patterns.keySet(), outFile); // else // throw new RuntimeException("why is the answer label null?"); return(finalwords); } else { if (constVars.wordScoring.Equals(GetPatternsFromDataMultiClass.WordScoring.Bpb)) { Counters.AddInPlace(terms, wordsPatExtracted); ICounter <CandidatePhrase> maxPatWeightTerms = new ClassicCounter <CandidatePhrase>(); IDictionary <CandidatePhrase, E> wordMaxPat = new Dictionary <CandidatePhrase, E>(); foreach (KeyValuePair <CandidatePhrase, ClassicCounter <E> > en in terms.EntrySet()) { ICounter <E> weights = new ClassicCounter <E>(); foreach (E k in en.Value.KeySet()) { weights.SetCount(k, patternsLearnedThisIter.GetCount(k)); } maxPatWeightTerms.SetCount(en.Key, Counters.Max(weights)); wordMaxPat[en.Key] = Counters.Argmax(weights); } Counters.RemoveKeys(maxPatWeightTerms, alreadyIdentifiedWords); double maxvalue = Counters.Max(maxPatWeightTerms); ICollection <CandidatePhrase> words = Counters.KeysAbove(maxPatWeightTerms, maxvalue - 1e-10); CandidatePhrase bestw = null; if (words.Count > 1) { double max = double.NegativeInfinity; foreach (CandidatePhrase w in words) { if (terms.GetCount(w, wordMaxPat[w]) > max) { max = terms.GetCount(w, wordMaxPat[w]); bestw = w; } } } else { if (words.Count == 1) { bestw = words.GetEnumerator().Current; } else { return(new ClassicCounter <CandidatePhrase>()); } } Redwood.Log(ConstantsAndVariables.minimaldebug, "Selected Words: " + bestw); return(Counters.AsCounter(Arrays.AsList(bestw))); } else { throw new Exception("wordscoring " + constVars.wordScoring + " not identified"); } } }
/* * public void applyPats(Counter<E> patterns, String label, boolean computeDataFreq, TwoDimensionalCounter<Pair<String, String>, Integer> wordsandLemmaPatExtracted, * CollectionValuedMap<Integer, Triple<String, Integer, Integer>> matchedTokensByPat) throws ClassNotFoundException, IOException, InterruptedException, ExecutionException{ * Counter<E> patternsLearnedThisIterConsistsOnlyGeneralized = new ClassicCounter<E>(); * Counter<E> patternsLearnedThisIterRest = new ClassicCounter<E>(); * Set<String> specialWords = constVars.invertedIndex.getSpecialWordsList(); * List<String> extremelySmallStopWordsList = Arrays.asList(".",",","in","on","of","a","the","an"); * * for(Entry<Integer, Double> en: patterns.entrySet()){ * Integer pindex = en.getKey(); * SurfacePattern p = constVars.getPatternIndex().get(pindex); * String[] n = p.getSimplerTokensNext(); * String[] pr = p.getSimplerTokensPrev(); * boolean rest = false; * if(n!=null){ * for(String e: n){ * if(!specialWords.contains(e)){ * rest = true; * break; * } * } * } * if(rest == false && pr!=null){ * for(String e: pr){ * if(!specialWords.contains(e) && !extremelySmallStopWordsList.contains(e)){ * rest = true; * break; * } * } * } * if(rest) * patternsLearnedThisIterRest.setCount(en.getKey(), en.getValue()); * else * patternsLearnedThisIterConsistsOnlyGeneralized.setCount(en.getKey(), en.getValue()); * } * * * * Map<String, Set<String>> sentidswithfilerest = constVars.invertedIndex.getFileSentIdsFromPats(patternsLearnedThisIterRest.keySet(), constVars.getPatternIndex()); * * if (constVars.batchProcessSents) { * List<File> filesToLoad; * if(patternsLearnedThisIterConsistsOnlyGeneralized.size() > 0) * filesToLoad = Data.sentsFiles; * else{ * filesToLoad = new ArrayList<File>(); * for (String fname : sentidswithfilerest.keySet()) { * String filename; * // if(!constVars.usingDirForSentsInIndex) * // filename = constVars.saveSentencesSerDir+"/"+fname; * // else * filename = fname; * filesToLoad.add(new File(filename)); * } * } * * for (File fname : filesToLoad) { * Redwood.log(Redwood.DBG, "Applying patterns to sents from " + fname); * Map<String, List<CoreLabel>> sents = IOUtils.readObjectFromFile(fname); * * if(sentidswithfilerest != null && !sentidswithfilerest.isEmpty()){ * * String filename; * // if(constVars.usingDirForSentsInIndex) * // filename = constVars.saveSentencesSerDir+"/"+fname.getName(); * // else * filename = fname.getAbsolutePath(); * * Set<String> sentIDs = sentidswithfilerest.get(filename); * if (sentIDs != null){ * this.runParallelApplyPats(sents, sentIDs, label, patternsLearnedThisIterRest, wordsandLemmaPatExtracted, matchedTokensByPat); * } else * Redwood.log(Redwood.DBG, "No sentIds for " + filename + " in the index for the keywords from the patterns! The index came up with these files: " + sentidswithfilerest.keySet()); * } * if(patternsLearnedThisIterConsistsOnlyGeneralized.size() > 0){ * this.runParallelApplyPats(sents, sents.keySet(), label, patternsLearnedThisIterConsistsOnlyGeneralized, wordsandLemmaPatExtracted, matchedTokensByPat); * } * * if (computeDataFreq){ * Data.computeRawFreqIfNull(sents, constVars.numWordsCompound); * Data.fileNamesUsedToComputeRawFreq.add(fname.getName()); * } * } * * //Compute Frequency from the files not loaded using the invertedindex query. otherwise, later on there is an error. * if(computeDataFreq){ * for(File f: Data.sentsFiles){ * if(!Data.fileNamesUsedToComputeRawFreq.contains(f.getName())){ * Map<String, List<CoreLabel>> sents = IOUtils.readObjectFromFile(f); * Data.computeRawFreqIfNull(sents, constVars.numWordsCompound); * Data.fileNamesUsedToComputeRawFreq.add(f.getName()); * } * } * } * * } else { * * if (sentidswithfilerest != null && !sentidswithfilerest.isEmpty()) { * String filename = CollectionUtils.toList(sentidswithfilerest.keySet()).get(0); * Set<String> sentids = sentidswithfilerest.get(filename); * if (sentids != null) { * this.runParallelApplyPats(Data.sents, sentids, label, patternsLearnedThisIterRest, wordsandLemmaPatExtracted, matchedTokensByPat); * } else * throw new RuntimeException("How come no sentIds for " + filename + ". Index keyset is " + constVars.invertedIndex.getKeySet()); * } * if(patternsLearnedThisIterConsistsOnlyGeneralized.size() > 0){ * this.runParallelApplyPats(Data.sents, Data.sents.keySet(), label, patternsLearnedThisIterConsistsOnlyGeneralized, wordsandLemmaPatExtracted, matchedTokensByPat); * } * Data.computeRawFreqIfNull(Data.sents, constVars.numWordsCompound); * } * Redwood.log(Redwood.DBG, "# words/lemma and pattern pairs are " + wordsandLemmaPatExtracted.size()); * } */ private void StatsWithoutApplyingPatterns(IDictionary <string, DataInstance> sents, PatternsForEachToken patternsForEachToken, ICounter <E> patternsLearnedThisIter, TwoDimensionalCounter <CandidatePhrase, E> wordsandLemmaPatExtracted) { foreach (KeyValuePair <string, DataInstance> sentEn in sents) { IDictionary <int, ICollection <E> > pat4Sent = patternsForEachToken.GetPatternsForAllTokens(sentEn.Key); if (pat4Sent == null) { throw new Exception("How come there are no patterns for " + sentEn.Key); } foreach (KeyValuePair <int, ICollection <E> > en in pat4Sent) { CoreLabel token = null; ICollection <E> p1 = en.Value; // Set<Integer> p1 = en.getValue().first(); // Set<Integer> p2 = en.getValue().second(); // Set<Integer> p3 = en.getValue().third(); foreach (E index in patternsLearnedThisIter.KeySet()) { if (p1.Contains(index)) { if (token == null) { token = sentEn.Value.GetTokens()[en.Key]; } wordsandLemmaPatExtracted.IncrementCount(CandidatePhrase.CreateOrGet(token.Word(), token.Lemma()), index); } } } } }
public virtual ICounter <CandidatePhrase> ChooseTopWords(ICounter <CandidatePhrase> newdt, TwoDimensionalCounter <CandidatePhrase, E> terms, ICounter <CandidatePhrase> useThresholdNumPatternsForTheseWords, ICollection <CandidatePhrase> ignoreWords , double thresholdWordExtract) { IEnumerator <CandidatePhrase> termIter = Counters.ToPriorityQueue(newdt).GetEnumerator(); ICounter <CandidatePhrase> finalwords = new ClassicCounter <CandidatePhrase>(); while (termIter.MoveNext()) { if (finalwords.Size() >= constVars.numWordsToAdd) { break; } CandidatePhrase w = termIter.Current; if (newdt.GetCount(w) < thresholdWordExtract) { Redwood.Log(ConstantsAndVariables.extremedebug, "not adding word " + w + " and any later words because the score " + newdt.GetCount(w) + " is less than the threshold of " + thresholdWordExtract); break; } System.Diagnostics.Debug.Assert((newdt.GetCount(w) != double.PositiveInfinity)); if (useThresholdNumPatternsForTheseWords.ContainsKey(w) && NumNonRedundantPatterns(terms, w) < constVars.thresholdNumPatternsApplied) { Redwood.Log("extremePatDebug", "Not adding " + w + " because the number of non redundant patterns are below threshold of " + constVars.thresholdNumPatternsApplied + ":" + terms.GetCounter(w).KeySet()); continue; } CandidatePhrase matchedFuzzy = null; if (constVars.minLen4FuzzyForPattern > 0 && ignoreWords != null) { matchedFuzzy = ConstantsAndVariables.ContainsFuzzy(ignoreWords, w, constVars.minLen4FuzzyForPattern); } if (matchedFuzzy == null) { Redwood.Log("extremePatDebug", "adding word " + w); finalwords.SetCount(w, newdt.GetCount(w)); } else { Redwood.Log("extremePatDebug", "not adding " + w + " because it matched " + matchedFuzzy + " in common English word"); ignoreWords.Add(w); } } string nextTen = string.Empty; int n = 0; while (termIter.MoveNext()) { n++; if (n > 10) { break; } CandidatePhrase w = termIter.Current; nextTen += ";\t" + w + ":" + newdt.GetCount(w); } Redwood.Log(Redwood.Dbg, "Next ten phrases were " + nextTen); return(finalwords); }