public static bool PageFilter(WikiPageContent page, WikiPageFilter filter) { foreach (string m in filter.IgnoreMorphology) { if (page.morph.Contains(m)) { return(false); } } foreach (string m in filter.IgnoreValue) { if (page.value.Contains(m)) { return(false); } } string[] vals = page.value.Split('\n'); foreach (string m in filter.IgnoreValueText) { int valueContainsCount = 0; foreach (string v in vals) { if (v.Contains(m)) { valueContainsCount++; } } if (valueContainsCount == vals.Length) { return(false); } } foreach (string m in filter.IgnoreGiponims) { if (page.giponims.Contains(m)) { return(false); } } foreach (string m in filter.IgnoreGiperonims) { if (page.giperonims.Contains(m)) { return(false); } } return(true); }
public static Dictionary <string, WikiPageContent> ParsePages(List <WikiPage> pages, ref string progress, WikiPageFilter filter) { var dict = new Dictionary <string, WikiPageContent>(); int counter = 0; foreach (var page in pages) { if (!run) { break; } try { WikiPageContent currentPage = ParsePage(page.title); if (PageFilter(currentPage, filter)) { dict.Add(page.title, currentPage); } } catch (Exception) { } counter++; progress = String.Format("Downloading pages: {0}/{1}", counter, pages.Count); } progress = "finished"; return(dict); }