private static void GitHubReadmeExamples() { var tr = LanguageFactory.Create(LanguageType.Turkish); var solutions = tr.Analyze("yolsuzu"); foreach (var solution in solutions) { Console.WriteLine("\t{0}", solution); Console.WriteLine("\toriginal:{0} stem:{1} root:{2}\n", solution.GetSurface(), solution.GetStem().GetSurface(), solution.Root); //Stemming } //Method 1: Specify the ids of the morphemes that constitute the word var word1 = tr.Generate("kitap/ISIM", "IC_COGUL_lAr", "IC_SAHIPLIK_BEN_(U)m", "IC_HAL_BULUNMA_DA", "IC_AITLIK_ki", "IC_COGUL_lAr", "IC_HAL_AYRILMA_DAn"); //Method 2: Specify the string representation of the analysis of the word. var analysis = "kitap/ISIM IC_COGUL_lAr IC_SAHIPLIK_BEN_(U)m"; var word2 = tr.GetWord(analysis); Console.WriteLine(word1.GetSurface()); Console.WriteLine(word2.GetSurface()); }
public void Create_ShouldCreateLanguageModel(Db db, [Content] DbItem item) { var contentItem = db.GetItem(item.ID); Context.Item = contentItem; var language = LanguageFactory.Create(Context.Language); language.Should().NotBeNull(); language.TwoLetterCode.Should().BeEquivalentTo(Context.Language.Name); }
private static void AnalysisAndStemming() { var tr = LanguageFactory.Create(LanguageType.Turkish); var stems = tr.Analyze("ehemmiyetsiz").Select(s => s.Root.LexicalForm).ToList(); foreach (var stem in stems) { Console.WriteLine("\t{0}", stem); } }
private static void Generation() { var tr = LanguageFactory.Create(LanguageType.Turkish); var solutions = tr.Analyze("suyu"); var surfaces = solutions[0].GetSurfacesAfterEachPhase(); foreach (var surface in surfaces) { Console.WriteLine(surface); } }
public static void prepareDocuments(List <Document> documents, HashSet <string> classifierGrams) { var filenames = from fullFilename in Directory.EnumerateFiles("raw_texts/", "*.txt", SearchOption.AllDirectories) select Path.GetFullPath(fullFilename); List <string> list = filenames.ToList(); StreamReader sr2 = new StreamReader("stop_words.txt", Encoding.GetEncoding("ISO-8859-9")); Language tr = LanguageFactory.Create(LanguageType.Turkish); char[] seperators = { ' ', '\n', '\t', '\r', '\0' }; string[] stopWords = sr2.ReadToEnd().Split('\n'); Dictionary <string, int> allGrams = new Dictionary <string, int>(); List <Dictionary <string, int> > grams = new List <Dictionary <string, int> >(); Parallel.ForEach(list, i => { StreamReader sr = new StreamReader(i, Encoding.GetEncoding("ISO-8859-9")); string cleanData = PreProcessing.editFile(i, sr, sr2, tr, seperators, stopWords); var gram = CreateGramsFrequencies.make2Gram(cleanData).Concat(CreateGramsFrequencies.make3gram(cleanData)).ToDictionary(e => e.Key, e => e.Value);//2gram ile 3 grami birlestir grams.Add(gram); documents.Add(new Document(Path.GetFileNameWithoutExtension(i), Directory.GetParent(i).Name, gram)); sr.Close(); }); foreach (var i in grams) { foreach (KeyValuePair <string, int> ix in i) { if (!allGrams.ContainsKey(ix.Key)) { allGrams.Add(ix.Key, ix.Value); } else { allGrams[ix.Key] += ix.Value; } } } foreach (KeyValuePair <string, int> gram in allGrams) { if (gram.Value >= 50) { if (!classifierGrams.Contains(gram.Key)) { classifierGrams.Add(gram.Key); } } } }
string CekimEkleriniTemizle(string metin) // çekim eklerini temizler { string[] kelimeler = metin.Split(' '); //metindeki her kelimeyi alır Language tr = LanguageFactory.Create(LanguageType.Turkish); //nuve kütüphanesi string sonhali = ""; for (int i = 0; i < kelimeler.Length; i++) { IList <Word> kokler = tr.Analyze(kelimeler[i]); // kelimenin analizi if (kokler.Count > 0) // sonuc varsa { sonhali += kokler[0].GetStem().GetSurface() + "_"; // kelimenin govdesini al(GetStem) ve stringe çevir(GetSurface) } } return(sonhali); }
public static string[] ReplaceRoots(string root, string[] words) { Language turkish = LanguageFactory.Create(LanguageType.Turkish); var replacedWords = new List <string>(); foreach (string word in words) { IEnumerable <Word> solutions = turkish.Analyze(word); foreach (Word solution in solutions) { string output = solution.GetSurface(); solution.Root = turkish.GetRootsHavingSurface(root).First(); output += "\t" + solution.GetSurface(); replacedWords.Add(output); } } return(replacedWords.ToArray()); }
private void KelimeKokleriniBul() { bool girdi = false; Language tr = LanguageFactory.Create(LanguageType.Turkish); IList <Word> words; foreach (string gelen in DosyaKelimeler) { girdi = false; //Console.WriteLine(gelen); words = tr.Analyze(gelen); if (words.Count != 0) { if (sozluk.ContainsKey(words[0].GetStem().ToString().Split('/')[0])) { sozluk[words[0].GetStem().ToString().Split('/')[0]]++; girdi = true; } if (!girdi) { sozluk.Add(words[0].GetStem().ToString().Split('/')[0], 1); girdi = false; } } else { if (sozluk.ContainsKey(gelen)) { sozluk[gelen]++; girdi = true; } if (!girdi) { sozluk.Add(gelen, 1); girdi = false; } } } }
public void Test() { var tr = LanguageFactory.Create(LanguageType.Turkish); var newTr = MutableLanguage.CopyFrom(tr); var entry = new RootEntry( lex: "başa gel", pos: "FIIL", surfaces: new[] { "başa gel" }, labels: new[] { "cverb" }, rules: Enumerable.Empty <string>()); Assert.True(newTr.TryAdd(entry)); var solutions = tr.Analyze("başa gelen"); Assert.AreEqual(0, solutions.Count); var solutionsExtendedTr = newTr.Analyze("başa gelen"); Assert.AreEqual(1, solutionsExtendedTr.Count); Assert.AreEqual("başa gel/FIIL FIILIMSI_SIFAT_(y)An", solutionsExtendedTr[0].Analysis); }
public static Language GetActive() { return(LanguageFactory.Create(Context.Language)); }
public void tokenize(string path, int classIndex) { StreamReader r = new StreamReader(path, Encoding.GetEncoding("windows-1254")); Language tr = LanguageFactory.Create(LanguageType.Turkish); string line; string[] rawWords; string[] token; string analyzMax = ""; int analyzMaxL = Int32.MinValue; string temp; int tempL; int docMax = Int32.MinValue; int docMax2 = Int32.MinValue; int analyzL; int tokenL; Dictionary <string, int> document = new Dictionary <string, int>(); while (!r.EndOfStream)//tokenizer kısımları { line = r.ReadLine(); rawWords = Regex.Split(line, @"((\)('|’)\w+)|\W('|’)|('|’)\W|^('|’)|$('|’)|\d+('|’)\w+|\d+\w+|\d+[^a-zA-Z ]+\w+|\w+\d+|\d+|(\)|\())|[^\w('|’)]", RegexOptions.ExplicitCapture | RegexOptions.Compiled); //Büyülü regex'e göre metini parçalara ayırır foreach (string w in rawWords) { //boş stringler ve istenmeyen bazı durumlar da -örneğin sayılar- atılır if (w != "" && Regex.IsMatch(w, @"\D\w", RegexOptions.Compiled)) { analyzMaxL = Int32.MinValue; analyzMax = ""; //daha çok büyülü regex token = Regex.Split(w, @"(\W*)('|’)(\w+|\W+)", RegexOptions.ExplicitCapture | RegexOptions.Compiled); token[0] = token[0].ToLower(); IList <Word> solutions = tr.Analyze(token[0]);//morfolojik analiz foreach (var solution in solutions) { temp = solution.GetStem().GetSurface(); //Stemming tempL = temp.Length; //Genel olarak köke ne kadar az yaklaşırsa metinle o kadar alakalı //olduğunu tespit ettik bu yüzden en uzun stemi aldık if (tempL > analyzMaxL) { analyzMaxL = tempL; analyzMax = temp; } } analyzL = analyzMax.Length; tokenL = token[0].Length; //stop words leri eliyor if (analyzMax != "") { if (analyzL > 2 && !stopWords.Contains(analyzMax)) { if (corpus.ContainsKey(analyzMax)) { corpus[analyzMax]++; } else { corpus.Add(analyzMax, 1); wordIndex.Add(analyzMax, Index); Index++; } if (document.ContainsKey(analyzMax)) { document[analyzMax]++; } else { document.Add(analyzMax, 1); } } } else { if (token[0].Length > 2 && !stopWords.Contains(token[0])) { if (corpus.ContainsKey(token[0])) { corpus[token[0]]++; } else { corpus.Add(token[0], 1); wordIndex.Add(token[0], Index); Index++; } if (document.ContainsKey(token[0])) { document[token[0]]++; } else { document.Add(token[0], 1); } } } if (analyzL > 2 && !stopWords.Contains(analyzMax)) { if (document[analyzMax] > docMax) { docMax2 = docMax; docMax = document[analyzMax]; } else if (document[analyzMax] > docMax2) { docMax2 = document[analyzMax]; } else { if (tokenL > 2 && !stopWords.Contains(token[0]) && analyzMax == "") { if (document[token[0]] > docMax) { docMax2 = docMax; docMax = document[token[0]]; } else if (document[token[0]] > docMax2) { docMax2 = document[token[0]]; } } } } } } } DocumentMax.Add(new int[] { docMax, docMax2 }); DocumentClassIndex.Add(classIndex); DocumentWordFreq.Add(document); }