public override bool Check(ArticleInfo article) { foreach (Regex r in Contains) { if (!r.IsMatch(article.Text)) return false; } return true; }
public override bool Check(ArticleInfo article) { DateTime timestamp; if (DateTime.TryParse(article.Timestamp, out timestamp)) { return((DateTime.Compare(timestamp, From) >= 0) && (DateTime.Compare(timestamp, To) <= 0)); } return(false); }
public override bool Check(ArticleInfo article) { foreach (Regex r in Contains) { if (!r.IsMatch(article.Text)) { return(false); } } return(true); }
private void ScanArticle(ArticleInfo ai) { foreach (Scan z in Scanners) { if (!z.Check(ref ai.Text, ref ai.Title, ai.Timestamp, ai.Restrictions)) { return; } } OutputQueue.Add(ai.Title); }
public override bool Check(ArticleInfo article) { Actual = WikiRegexes.SimpleWikiLink.Matches(article.Text).Count; if (M == MoreLessThan.MoreThan) { return(Actual > Test); } if (M == MoreLessThan.LessThan) { return(Actual < Test); } return(Actual == Test); }
public override bool Check(ArticleInfo article) { Actual = Tools.WordCount(article.Text); if (M == MoreLessThan.MoreThan) { return(Actual > Test); } if (M == MoreLessThan.LessThan) { return(Actual < Test); } return(Actual == Test); }
public override bool Check(ArticleInfo article) { foreach (Match m in WikiRegexes.SimpleWikiLink.Matches(article.Text)) { string y = System.Web.HttpUtility.UrlDecode(m.Value.Replace("+", "%2B")); if (m.Value != y) { return(false); } } return(true); }
public override bool Check(ArticleInfo article) { Actual = article.Text.Length; if (M == MoreLessThan.MoreThan) { return(Actual > Test); } if (M == MoreLessThan.LessThan) { return(Actual < Test); } return(Actual == Test); }
public override bool Check(ArticleInfo article) { if (WikiRegexes.Category.IsMatch(article.Text)) { return(false); } foreach (Match m in WikiRegexes.Template.Matches(article.Text)) { if (!m.Value.Contains("stub")) { return(false); } } return(true); }
private void ScanArticle(ArticleInfo ai) { if (IgnoreComments) { ai.Text = WikiRegexes.Comments.Replace(ai.Text, ""); } foreach (Scan z in Scanners) { if (!z.Check(ai)) { return; } } OutputQueue.Add(ai.Title); }
private void SecondaryThread() { try { bool sleep; int sleeps = 0; while (Run) { if (PendingArticles.Count > 0) { lock (PendingArticles) { if (PendingArticles.Count > 0) { ArticleInfo ai = PendingArticles.Remove(); ScanArticle(ai); sleep = false; } else { sleep = true; } } } else { sleep = true; } if (sleep) { Thread.Sleep(1); sleeps++; } } //System.Windows.Forms.MessageBox.Show(sleeps.ToString()); } catch (ThreadAbortException) { } catch (Exception ex) { //TODO: ErrorHandler.Handle(ex); } }
public override bool Check(ArticleInfo article) { foreach (Match m in WikiRegexes.PipedWikiLink.Matches(article.Text)) { string a = m.Groups[1].Value; string b = m.Groups[2].Value; if (a == b || Tools.TurnFirstToLower(a) == b) { return(true); } if (a + "s" == b || Tools.TurnFirstToLower(a) + "s" == b) { return(true); } } return(false); }
public override bool Check(ArticleInfo article) { bool restrictionStringEmpty = (string.IsNullOrEmpty(article.Restrictions)); bool noEditRestriction = string.IsNullOrEmpty(Edit); bool noMoveRestriction = string.IsNullOrEmpty(Move); if (restrictionStringEmpty) { return(noEditRestriction && noMoveRestriction); } if (!noEditRestriction && !article.Restrictions.Contains(EditRest + Edit)) { return(false); } return(noMoveRestriction || article.Restrictions.Contains(MoveRest + Move)); }
private void SecondaryThread() { try { while (Run) { bool sleep; if (PendingArticles.Count > 0) { lock (PendingArticles) { if (PendingArticles.Count > 0) { ArticleInfo ai = PendingArticles.Remove(); ScanArticle(ai); sleep = false; } else { sleep = true; } } } else { sleep = true; } if (sleep) { Thread.Sleep(1); } } } catch (ThreadAbortException) { } catch (Exception ex) { ErrorHandler.Handle(ex); } }
/// <summary> /// Reads a page from the reader, returns ArticleInfo or null if EOF /// </summary> private ArticleInfo ReadArticle(XmlReader reader) { do { if (!reader.ReadToFollowing("page")) { return(null); } }while (!reader.IsStartElement()); ArticleInfo ai = new ArticleInfo(); while (reader.Read() && reader.Name != "page") // stop on closing element { if (!reader.IsStartElement()) { continue; } switch (reader.Name) { case "title": ai.Title = reader.ReadString(); break; case "timestamp": ai.Timestamp = reader.ReadString(); break; case "restrictions": ai.Restrictions = reader.ReadString(); break; case "text": ai.Text = reader.ReadString(); break; } } return(ai.IsFullyRead ? ai : null); }
public override bool Check(ArticleInfo article) { return(!Tools.IsRedirect(article.Text)); }
public override bool Check(ArticleInfo article) { return(BulletRegex.IsMatch(article.Text)); }
public override bool Check(ArticleInfo article) { foreach (Match m in WikiRegexes.SimpleWikiLink.Matches(article.Text)) { string y = System.Web.HttpUtility.UrlDecode(m.Value.Replace("+", "%2B")); if (m.Value != y) return false; } return true; }
public override bool Check(ArticleInfo article) { int bracketLength = 0; return(Parsers.UnbalancedBrackets(article.Text, ref bracketLength) != -1); }
public override bool Check(ArticleInfo article) { return(!base.Check(article)); }
/// <summary> /// Reads a page from the reader, returns ArticleInfo or null if EOF /// </summary> private ArticleInfo ReadArticle(XmlReader reader) { do if (!reader.ReadToFollowing("page")) return null; while (!reader.IsStartElement()); ArticleInfo ai = new ArticleInfo(); while (reader.Read() && reader.Name != "page") // stop on closing element { if (!reader.IsStartElement()) continue; switch (reader.Name) { case "title": ai.Title = reader.ReadString(); break; case "timestamp": ai.Timestamp = reader.ReadString(); break; case "restrictions": ai.Restrictions = reader.ReadString(); break; case "text": ai.Text = reader.ReadString(); break; } } return ai.IsFullyRead ? ai : null; }
public override bool Check(ArticleInfo article) { return (BulletRegex.IsMatch(article.Text)); }
public override bool Check(ArticleInfo article) { Parsers.ChangeToDefaultSort(article.Text, article.Title, out Skip); return !Skip; }
public override bool Check(ArticleInfo article) { foreach (Match m in WikiRegexes.PipedWikiLink.Matches(article.Text)) { string a = m.Groups[1].Value; string b = m.Groups[2].Value; if (a == b || Tools.TurnFirstToLower(a) == b) return true; if (a + "s" == b || Tools.TurnFirstToLower(a) + "s" == b) return true; } return false; }
public override bool Check(ArticleInfo article) { Parsers.FixHeadings(article.Text, article.Title, out Skip); return !Skip; }
public override bool Check(ArticleInfo article) { P.Unicodify(article.Text, out Skip); return !Skip; }
public virtual bool Check(ArticleInfo article) { return true; }
public override bool Check(ArticleInfo article) { int bracketLength = 0; return (Parsers.UnbalancedBrackets(article.Text, ref bracketLength) != -1); }
public override bool Check(ArticleInfo article) { return(Retf.DetectTypo(article.Text, article.Title)); }
public override bool Check(ArticleInfo article) { return Retf.DetectTypo(article.Text, article.Title); }
public override bool Check(ArticleInfo article) { Actual = Tools.WordCount(article.Text); if (M == MoreLessThan.MoreThan) return (Actual > Test); if (M == MoreLessThan.LessThan) return (Actual < Test); return (Actual == Test); }
public override bool Check(ArticleInfo article) { return (!Tools.IsRedirect(article.Text)); }
private void Process() { string articleTitle = ""; try { using (XmlTextReader reader = new XmlTextReader(stream)) { reader.WhitespaceHandling = WhitespaceHandling.None; if (From.Length > 0) { //move to start from article while (reader.Read() && mRun) { if (reader.Name == "page") { reader.ReadToFollowing("title"); articleTitle = reader.ReadString(); if (From == articleTitle) { break; } } } } while (reader.Read() && mRun) { if (reader.Name == "page") { ArticleInfo ai = new ArticleInfo(); reader.ReadToFollowing("title"); ai.Title = articleTitle = reader.ReadString(); //reader.ReadToFollowing(restriction); //TODO:This is wrong. Only want to read the restriction if in that <page></page> if (reader.Name == "restrictions") { ai.Restrictions = reader.ReadString(); } else { ai.Restrictions = ""; } reader.ReadToFollowing("timestamp"); ai.Timestamp = reader.ReadString(); reader.ReadToFollowing("text"); ai.Text = reader.ReadString(); if (IgnoreComments) { ai.Text = WikiRegexes.Comments.Replace(ai.Text, ""); } if (MultiThreaded) { if (PendingArticles.Count < ProcessorCount * 4 + 5) { PendingArticles.Add(ai); } else { ScanArticle(ai); } } else { ScanArticle(ai); } } } lock (ScanThread) { stream = null; } if (MultiThreaded) { while (PendingArticles.Count > 0) { Thread.Sleep(10); } mRun = false; foreach (Thread thr in SecondaryThreads) { thr.Join(); } } } } catch (ThreadAbortException) { } catch (Exception ex) { if (boolMessage) { //System.Windows.Forms.MessageBox.Show("Problem on " + articleTitle + "\r\n\r\n" + ex.Message); ErrorHandler.Handle(ex); } } finally { if (boolMessage) { context.Post(SOPCstopped, articleTitle); } } }
public override bool Check(ArticleInfo article) { DateTime timestamp; if (DateTime.TryParse(article.Timestamp, out timestamp)) return ((DateTime.Compare(timestamp, From) >= 0) && (DateTime.Compare(timestamp, To) <= 0)); return false; }
public override bool Check(ArticleInfo article) { Actual = article.Text.Length; if (M == MoreLessThan.MoreThan) return (Actual > Test); if (M == MoreLessThan.LessThan) return (Actual < Test); return (Actual == Test); }
public override bool Check(ArticleInfo article) { bool restrictionStringEmpty = (string.IsNullOrEmpty(article.Restrictions)); bool noEditRestriction = string.IsNullOrEmpty(Edit); bool noMoveRestriction = string.IsNullOrEmpty(Move); if (restrictionStringEmpty) { return (noEditRestriction && noMoveRestriction); } if (!noEditRestriction && !article.Restrictions.Contains(EditRest + Edit)) return false; return noMoveRestriction || article.Restrictions.Contains(MoveRest + Move); }
public override bool Check(ArticleInfo article) { P.FixPeopleCategories(article.Text, article.Title, false, out Skip); return !Skip; }
public override bool Check(ArticleInfo article) { if (WikiRegexes.Category.IsMatch(article.Text)) return false; foreach (Match m in WikiRegexes.Template.Matches(article.Text)) { if (!m.Value.Contains("stub")) return false; } return true; }
public override bool Check(ArticleInfo article) { P.Unicodify(article.Text, out Skip); return(!Skip); }
public override bool Check(ArticleInfo article) { return !base.Check(article); }
public override bool Check(ArticleInfo article) { Parsers.FixHeadings(article.Text, article.Title, out Skip); return(!Skip); }
public override bool Check(ArticleInfo article) { P.CiteTemplateDates(article.Text, out Skip); return !Skip; }
public override bool Check(ArticleInfo article) { Parsers.LivingPeople(article.Text, out Skip); return(!Skip); }
public override bool Check(ArticleInfo article) { Parsers.ChangeToDefaultSort(article.Text, article.Title, out Skip); return(!Skip); }
public override bool Check(ArticleInfo article) { P.BoldTitle(article.Text, article.Title, out Skip); return !Skip; }
public override bool Check(ArticleInfo article) { return (Namespaces.Contains(Namespace.Determine(article.Title))); }
public override bool Check(ArticleInfo article) { return (Contains.IsMatch(article.Title)); }
public override bool Check(ArticleInfo article) { Parsers.LivingPeople(article.Text, out Skip); return !Skip; }
public override bool Check(ArticleInfo article) { P.BoldTitle(article.Text, article.Title, out Skip); return(!Skip); }
private void Process() { string articleTitle = ""; try { using (XmlTextReader reader = new XmlTextReader(stream)) { reader.WhitespaceHandling = WhitespaceHandling.None; if (From.Length > 0) { //move to start from article while (reader.Read() && mRun) { if (reader.Name == "page") { reader.ReadToFollowing("title"); articleTitle = reader.ReadString(); if (From == articleTitle) break; } } } while (reader.Read() && mRun) { if (reader.Name == "page") { ArticleInfo ai = new ArticleInfo(); reader.ReadToFollowing("title"); ai.Title = articleTitle = reader.ReadString(); //reader.ReadToFollowing(restriction); //TODO:This is wrong. Only want to read the restriction if in that <page></page> if (reader.Name == "restrictions") ai.Restrictions = reader.ReadString(); else ai.Restrictions = ""; reader.ReadToFollowing("timestamp"); ai.Timestamp = reader.ReadString(); reader.ReadToFollowing("text"); ai.Text = reader.ReadString(); if (IgnoreComments) ai.Text = WikiRegexes.Comments.Replace(ai.Text, ""); if (MultiThreaded) { if (PendingArticles.Count < ProcessorCount * 4 + 5) { PendingArticles.Add(ai); } else { ScanArticle(ai); } } else { ScanArticle(ai); } } } lock (ScanThread) { stream = null; } if (MultiThreaded) { while (PendingArticles.Count > 0) Thread.Sleep(10); mRun = false; foreach (Thread thr in SecondaryThreads) thr.Join(); } } } catch (ThreadAbortException) { } catch (Exception ex) { if (boolMessage) //System.Windows.Forms.MessageBox.Show("Problem on " + articleTitle + "\r\n\r\n" + ex.Message); ErrorHandler.Handle(ex); } finally { if (boolMessage) context.Post(SOPCstopped, articleTitle); } }
public override bool Check(ArticleInfo article) { P.CiteTemplateDates(article.Text, out Skip); return(!Skip); }
public override bool Check(ArticleInfo article) { Actual = WikiRegexes.SimpleWikiLink.Matches(article.Text).Count; if (M == MoreLessThan.MoreThan) return (Actual > Test); if (M == MoreLessThan.LessThan) return (Actual < Test); return (Actual == Test); }
public override bool Check(ArticleInfo article) { string articleTextAfter = Parsers.ReorderReferences(article.Text); return(!articleTextAfter.Equals(article.Text)); }
private void ScanArticle(ArticleInfo ai) { if (IgnoreComments) ai.Text = WikiRegexes.Comments.Replace(ai.Text, ""); foreach (Scan z in Scanners) { if (!z.Check(ai)) { return; } } OutputQueue.Add(ai.Title); }
public virtual bool Check(ArticleInfo article) { return(true); }
private void Process() { string articleTitle = ""; try { using (XmlTextReader reader = new XmlTextReader(stream)) { reader.WhitespaceHandling = WhitespaceHandling.None; if (From.Length > 0) { //move to start from article while (Run && reader.Read()) { if (reader.NodeType != XmlNodeType.Element) { continue; } if (reader.Name != "title") { reader.ReadToFollowing("page"); continue; } //reader.ReadToFollowing("title"); articleTitle = reader.ReadString(); if (From == articleTitle) { break; } } } while (Run) { ArticleInfo ai = ReadArticle(reader); if (ai == null) { break; } articleTitle = ai.Title; // we must maintain a huge enough buffer to safeguard against fluctuations // of page size if (MultiThreaded && (PendingArticles.Count < ProcessorCount * 10)) { PendingArticles.Add(ai); } else { ScanArticle(ai); } } lock (ScanThread) { stream = null; } if (MultiThreaded) { while (PendingArticles.Count > 0) { Thread.Sleep(10); } Run = false; foreach (Thread thr in SecondaryThreads) { thr.Join(); } } } } catch (ThreadAbortException) { } catch (Exception ex) { if (Message) { //System.Windows.Forms.MessageBox.Show("Problem on " + articleTitle + "\r\n\r\n" + ex.Message); ErrorHandler.Handle(ex); } } finally { if (Message) { Context.Post(SOPCstopped, articleTitle); } } }
public override bool Check(ArticleInfo article) { P.FixPeopleCategories(article.Text, article.Title, false, out Skip); return(!Skip); }
public override bool Check(ArticleInfo article) { return Parsers.MissingDefaultSort(article.Text, article.Title); }
public override bool Check(ArticleInfo article) { string articleTextAfter = Parsers.ReorderReferences(article.Text); return !articleTextAfter.Equals(article.Text); }