/// <summary> /// Learn Collocation pairs and triples according to patterns in Treebank-3 tagged files /// </summary> /// <remarks> /// the patterns are from: /// Justeson, John S., and Slava M. Katz. 1995. Technical terminology: some linguistic properties and /// an algorithm for identification in text. Natural Language Engineering 1:9-27. /// </remarks> /// <returns>IEnumerable of sorted frequencies (KeyValuePairs of collocated terms /// and the number of occurrrences of that string) of each collocation found in the corpus. /// </returns> public static IEnumerable<KeyValuePair<string, double>> collocated_terms_in_Treebank3() { var collocated_words_pattern = RegexTools.regex_filter_pattern("<J\\S+|N\\S+><J\\S+|N\\S+|IN\\S|TO\\S>*<N\\S+>"); //Console.WriteLine(collocated_words_pattern); var treebank3 = new Treebank3CorpusReader(Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), @"Data\Treebank-3")); foreach (var content in treebank3.read_tagged_sents() .Select((x) => Regex.Match(x, collocated_words_pattern).Groups[0].Value) .Select((x) => TextTools.get_term_from_string(x).DefaultIfEmpty("").Aggregate((a, b) => a + " " + b)) .Freqs().Generate().OrderBy((x) => x.Value)) { yield return content; } }
private static void most_common_collocated_NPs_in_Treebank3() { var collocated_words_pattern = RegexTools.regex_filter_pattern("<J\\S+|N\\S+><J\\S+|N\\S+|IN\\S|TO\\S>*<N\\S+>"); Console.WriteLine(collocated_words_pattern); var treebank3 = new Treebank3CorpusReader(Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), @"Data\Treebank-3")); foreach (var content in treebank3.read_tagged_sents() .Select((x)=>Regex.Match(x, collocated_words_pattern).Groups[0].Value) .Select((x) => TextTools.get_term_from_string(x).DefaultIfEmpty("").Aggregate((a, b) => a + " " + b)) .Freqs().Generate().OrderBy((x)=>x.Value).Reverse().Take(1000)) { Console.WriteLine("{0} : {1}", content.Key, content.Value); } }