public P_c ( CategoryProbabilityDistribution trainingDistribution, |
||
trainingDistribution | CategoryProbabilityDistribution | |
testData | ||
n | int | |
prob_c | double | |
return | double |
public void Start(string mainFolder, string outputDir, ISmoothingTechnique smoothing, int nGramSize, ILanguageProcessor processor) { var resultSet = new ResultSet(); var bayesClassifier = new BayesTextClassifier(); var docReader = new ReadDocumentFromTiraFile(); var categories = new List<TextSource>(); Console.WriteLine("Scanning..."); dynamic jsonConfig; using (StreamReader sr = new StreamReader(Path.Combine(mainFolder, "meta-file.json"))) { jsonConfig = JsonConvert.DeserializeObject(sr.ReadToEnd()); } var unknownFolder = (string)jsonConfig.folder; var encodingString = (string)jsonConfig.encoding; var cultureString = (string)jsonConfig.language; CultureInfo ci = null; switch (cultureString) { case "EN": ci = new CultureInfo("en-US"); break; default: throw new ApplicationException("culture not found"); } Encoding encoding = null; switch (encodingString) { case "UTF8": encoding = Encoding.UTF8; break; case "ASCII": encoding = Encoding.ASCII; break; default: throw new ApplicationException("encoding not found"); } foreach(var item in jsonConfig["candidate-authors"]) { var authorName = (string)item["author-name"]; var docs = new DirectoryInfo(Path.Combine(mainFolder, authorName)).GetFiles(); var dataSource = new List<string>(); foreach (var doc in docs) { try { dataSource.Add(docReader.ReadDocumentText(doc.FullName, encoding, ci)); } catch { Console.WriteLine("Document {0} unreadable", doc.FullName); } } categories.Add(processor.Process(dataSource, authorName)); } int n = nGramSize; Console.WriteLine("Scanned {1} documents in {0} categories", categories.Count, categories.Select(el => el.Documents.Count).Aggregate((el1, el2) => el1 + el2)); var allInOne = new TextSource(); allInOne.Documents.AddRange(categories.SelectMany(el => el.Documents)); Console.WriteLine("Building hash tables ..", n); Parallel.ForEach(categories, category => { for (int i = 1; i <= n; i++) { category.BuildSegmentTable(i); Console.WriteLine("hashed {0} with n={1}", category.Name, i); } }); allInOne.SetNGramCache(NGramCache.Aggregate(categories.Select(el => el.GetNGramCache()))); Console.WriteLine("aggregated hashing"); Console.WriteLine("Getting smoothing ready .."); smoothing.Init(allInOne, nGramSize); var categoriesToTest = new Dictionary<TextSource, CategoryProbabilityDistribution>(); foreach (var cat in categories) { categoriesToTest[cat] = new CategoryProbabilityDistribution(cat, smoothing, n); } Console.WriteLine("Start classifying .."); int totalProgress = jsonConfig["unknown-texts"].Count * categoriesToTest.Count; int progress = 0; System.Timers.Timer t = new System.Timers.Timer(5000); t.Elapsed += (sender, eventArgs) => { Console.Title = "Task is Running. Progress: " + Math.Round((((double)progress / (double)totalProgress) * 100.0), 2).ToString() + "%"; }; t.AutoReset = true; t.Start(); foreach (var item in jsonConfig["unknown-texts"]) { TextSource topCategory = null; var maxProb = 0.0; var textName = (string)item["unknown-text"]; var probs = new List<double>(); Parallel.ForEach(categoriesToTest, catDist => { var docText = new string[] { docReader.ReadDocumentText(Path.Combine(mainFolder, unknownFolder, textName), encoding, ci) }; var docSource = processor.Process(docText, "unknown").Documents.First(); double p = bayesClassifier.P_c(catDist.Value, docSource, n, (double)catDist.Key.Documents.Count / (double)allInOne.Documents.Count); lock (probs) { probs.Add(p); if (topCategory == null || p > maxProb) { topCategory = catDist.Key; maxProb = p; } } Interlocked.Increment(ref progress); }); // getting the score probs.Remove(maxProb); double pre_score = 0.0; double max_sub_score = 0.0; foreach (var p in probs) { var subScore = Math.Abs((maxProb - p) / maxProb) * Math.Pow(Math.E, 3); // normalized difference var eSubScore = Math.Exp(-subScore); pre_score += eSubScore; if (eSubScore > max_sub_score) max_sub_score = eSubScore; } double score = Math.Round(1.0 - (0.5 * (pre_score / probs.Count) + 0.5 * max_sub_score), 2); Console.WriteLine("Classified {0} as author {1} with score {2}", textName, topCategory.Name, score); resultSet.answers.Add(new Result(textName, topCategory.Name, score)); Console.WriteLine("writing data to file ..."); string data = JsonConvert.SerializeObject(resultSet, Formatting.Indented); using (StreamWriter sw = new StreamWriter(Path.Combine(outputDir, "answers.json"), false)) { sw.Write(data); sw.Flush(); } } }
public void Start(ILanguageProcessor processor, ISmoothingTechnique smoothing) { var baseDirectory = AppDomain.CurrentDomain.BaseDirectory; var bayesClassifier = new BayesTextClassifier(); var docReader = new ReadDocumentFromXmlFile(); var docPath = Path.Combine(baseDirectory, "data"); var authors = new DirectoryInfo(docPath).GetDirectories(); var categories = new List<TextSource>(); // Prepare data foreach (var item in authors) { var docs = item.GetFiles(); var dataSource = new List<string>(); foreach (var doc in docs) { try { dataSource.Add(docReader.ReadDocumentText(doc.FullName, Encoding.GetEncoding(1253), new CultureInfo("el-GR"))); } catch { Console.WriteLine("Document {0} unreadable", doc.FullName); } } categories.Add(processor.Process(dataSource, item.Name)); } Console.WriteLine("Scanned {1} documents in {0} categories", categories.Count, categories.Select(el => el.Documents.Count).Aggregate((el1, el2) => el1 + el2)); var testPath = Path.Combine(baseDirectory, "test"); var testAuthors = new DirectoryInfo(testPath).GetDirectories(); var allInOne = new TextSource(); allInOne.Documents.AddRange(categories.SelectMany(el => el.Documents)); // choose n from 1 to 4 for (int n = 1; n <= 4; n++) { Console.WriteLine("-----PREPARE for n = {0}", n); Console.WriteLine("Building hash tables ..", n); Parallel.ForEach(categories, category => { category.BuildSegmentTable(n); }); allInOne.SetNGramCache(NGramCache.Aggregate(categories.Select(el => el.GetNGramCache()))); Console.WriteLine("Getting smoothing ready .."); smoothing.Init(allInOne, n); var categoriesToTest = new Dictionary<TextSource, CategoryProbabilityDistribution>(); foreach(var cat in categories) { categoriesToTest[cat] = new CategoryProbabilityDistribution(cat, smoothing, n); } int rightClassified = 0; int wrongClassified = 0; Console.WriteLine("-----Algorithm starts now"); foreach (var testAuthor in testAuthors) { foreach (var testDocument in testAuthor.GetFiles()) { TextSource topCategory = null; var maxProb = 0.0; Parallel.ForEach(categoriesToTest, catDist => { var docText = new[] { docReader.ReadDocumentText(testDocument.FullName, Encoding.GetEncoding(1253), new CultureInfo("el-GR")) }; var docSource = processor.Process(docText, testAuthor.Name).Documents.First(); double p = bayesClassifier.P_c(catDist.Value, docSource, n, 1.0 / (double)categories.Count); if (topCategory == null || p > maxProb) { topCategory = catDist.Key; maxProb = p; } }); Console.WriteLine("Classified {0} as author {1} - {2}", testDocument.Name, topCategory.Name, topCategory.Name == testAuthor.Name ? "correct" : "incorrect"); if (topCategory.Name == testAuthor.Name) rightClassified++; else wrongClassified++; } } Console.WriteLine("-----SUMMARY"); Console.WriteLine("Success rate for n={0} is {1}\n", n, (double)rightClassified / ((double)rightClassified + (double)wrongClassified)); } }