public override bool Check(ArticleInfo article)
{
foreach (Regex r in Contains)
{
if (!r.IsMatch(article.Text))
return false;
}
return true;
}
public override bool Check(ArticleInfo article)
{
DateTime timestamp;
if (DateTime.TryParse(article.Timestamp, out timestamp))
{
return((DateTime.Compare(timestamp, From) >= 0) && (DateTime.Compare(timestamp, To) <= 0));
}
return(false);
}
public override bool Check(ArticleInfo article)
{
foreach (Regex r in Contains)
{
if (!r.IsMatch(article.Text))
{
return(false);
}
}
return(true);
}
private void ScanArticle(ArticleInfo ai)
{
foreach (Scan z in Scanners)
{
if (!z.Check(ref ai.Text, ref ai.Title, ai.Timestamp, ai.Restrictions))
{
return;
}
}
OutputQueue.Add(ai.Title);
}
public override bool Check(ArticleInfo article)
{
Actual = WikiRegexes.SimpleWikiLink.Matches(article.Text).Count;
if (M == MoreLessThan.MoreThan)
{
return(Actual > Test);
}
if (M == MoreLessThan.LessThan)
{
return(Actual < Test);
}
return(Actual == Test);
}
public override bool Check(ArticleInfo article)
{
Actual = Tools.WordCount(article.Text);
if (M == MoreLessThan.MoreThan)
{
return(Actual > Test);
}
if (M == MoreLessThan.LessThan)
{
return(Actual < Test);
}
return(Actual == Test);
}
public override bool Check(ArticleInfo article)
{
foreach (Match m in WikiRegexes.SimpleWikiLink.Matches(article.Text))
{
string y = System.Web.HttpUtility.UrlDecode(m.Value.Replace("+", "%2B"));
if (m.Value != y)
{
return(false);
}
}
return(true);
}
public override bool Check(ArticleInfo article)
{
Actual = article.Text.Length;
if (M == MoreLessThan.MoreThan)
{
return(Actual > Test);
}
if (M == MoreLessThan.LessThan)
{
return(Actual < Test);
}
return(Actual == Test);
}
public override bool Check(ArticleInfo article)
{
if (WikiRegexes.Category.IsMatch(article.Text))
{
return(false);
}
foreach (Match m in WikiRegexes.Template.Matches(article.Text))
{
if (!m.Value.Contains("stub"))
{
return(false);
}
}
return(true);
}
private void ScanArticle(ArticleInfo ai)
{
if (IgnoreComments)
{
ai.Text = WikiRegexes.Comments.Replace(ai.Text, "");
}
foreach (Scan z in Scanners)
{
if (!z.Check(ai))
{
return;
}
}
OutputQueue.Add(ai.Title);
}
private void SecondaryThread()
{
try
{
bool sleep;
int sleeps = 0;
while (Run)
{
if (PendingArticles.Count > 0)
{
lock (PendingArticles)
{
if (PendingArticles.Count > 0)
{
ArticleInfo ai = PendingArticles.Remove();
ScanArticle(ai);
sleep = false;
}
else
{
sleep = true;
}
}
}
else
{
sleep = true;
}
if (sleep)
{
Thread.Sleep(1);
sleeps++;
}
}
//System.Windows.Forms.MessageBox.Show(sleeps.ToString());
}
catch (ThreadAbortException)
{ }
catch (Exception ex)
{
//TODO:
ErrorHandler.Handle(ex);
}
}
public override bool Check(ArticleInfo article)
{
foreach (Match m in WikiRegexes.PipedWikiLink.Matches(article.Text))
{
string a = m.Groups[1].Value;
string b = m.Groups[2].Value;
if (a == b || Tools.TurnFirstToLower(a) == b)
{
return(true);
}
if (a + "s" == b || Tools.TurnFirstToLower(a) + "s" == b)
{
return(true);
}
}
return(false);
}
public override bool Check(ArticleInfo article)
{
bool restrictionStringEmpty = (string.IsNullOrEmpty(article.Restrictions));
bool noEditRestriction = string.IsNullOrEmpty(Edit);
bool noMoveRestriction = string.IsNullOrEmpty(Move);
if (restrictionStringEmpty)
{
return(noEditRestriction && noMoveRestriction);
}
if (!noEditRestriction && !article.Restrictions.Contains(EditRest + Edit))
{
return(false);
}
return(noMoveRestriction || article.Restrictions.Contains(MoveRest + Move));
}
private void SecondaryThread()
{
try
{
while (Run)
{
bool sleep;
if (PendingArticles.Count > 0)
{
lock (PendingArticles)
{
if (PendingArticles.Count > 0)
{
ArticleInfo ai = PendingArticles.Remove();
ScanArticle(ai);
sleep = false;
}
else
{
sleep = true;
}
}
}
else
{
sleep = true;
}
if (sleep)
{
Thread.Sleep(1);
}
}
}
catch (ThreadAbortException)
{ }
catch (Exception ex)
{
ErrorHandler.Handle(ex);
}
}
/// <summary>
/// Reads a page from the reader, returns ArticleInfo or null if EOF
/// </summary>
private ArticleInfo ReadArticle(XmlReader reader)
{
do
{
if (!reader.ReadToFollowing("page"))
{
return(null);
}
}while (!reader.IsStartElement());
ArticleInfo ai = new ArticleInfo();
while (reader.Read() && reader.Name != "page") // stop on closing element
{
if (!reader.IsStartElement())
{
continue;
}
switch (reader.Name)
{
case "title":
ai.Title = reader.ReadString();
break;
case "timestamp":
ai.Timestamp = reader.ReadString();
break;
case "restrictions":
ai.Restrictions = reader.ReadString();
break;
case "text":
ai.Text = reader.ReadString();
break;
}
}
return(ai.IsFullyRead ? ai : null);
}
public override bool Check(ArticleInfo article)
{
return(!Tools.IsRedirect(article.Text));
}
public override bool Check(ArticleInfo article)
{
return(BulletRegex.IsMatch(article.Text));
}
public override bool Check(ArticleInfo article)
{
foreach (Match m in WikiRegexes.SimpleWikiLink.Matches(article.Text))
{
string y = System.Web.HttpUtility.UrlDecode(m.Value.Replace("+", "%2B"));
if (m.Value != y)
return false;
}
return true;
}
public override bool Check(ArticleInfo article)
{
int bracketLength = 0;
return(Parsers.UnbalancedBrackets(article.Text, ref bracketLength) != -1);
}
public override bool Check(ArticleInfo article)
{
return(!base.Check(article));
}
/// <summary>
/// Reads a page from the reader, returns ArticleInfo or null if EOF
/// </summary>
private ArticleInfo ReadArticle(XmlReader reader)
{
do
if (!reader.ReadToFollowing("page")) return null;
while (!reader.IsStartElement());
ArticleInfo ai = new ArticleInfo();
while (reader.Read() && reader.Name != "page") // stop on closing element
{
if (!reader.IsStartElement()) continue;
switch (reader.Name)
{
case "title":
ai.Title = reader.ReadString();
break;
case "timestamp":
ai.Timestamp = reader.ReadString();
break;
case "restrictions":
ai.Restrictions = reader.ReadString();
break;
case "text":
ai.Text = reader.ReadString();
break;
}
}
return ai.IsFullyRead ? ai : null;
}
public override bool Check(ArticleInfo article)
{
return (BulletRegex.IsMatch(article.Text));
}
public override bool Check(ArticleInfo article)
{
Parsers.ChangeToDefaultSort(article.Text, article.Title, out Skip);
return !Skip;
}
public override bool Check(ArticleInfo article)
{
foreach (Match m in WikiRegexes.PipedWikiLink.Matches(article.Text))
{
string a = m.Groups[1].Value;
string b = m.Groups[2].Value;
if (a == b || Tools.TurnFirstToLower(a) == b)
return true;
if (a + "s" == b || Tools.TurnFirstToLower(a) + "s" == b)
return true;
}
return false;
}
public override bool Check(ArticleInfo article)
{
Parsers.FixHeadings(article.Text, article.Title, out Skip);
return !Skip;
}
public override bool Check(ArticleInfo article)
{
P.Unicodify(article.Text, out Skip);
return !Skip;
}
public virtual bool Check(ArticleInfo article)
{
return true;
}
public override bool Check(ArticleInfo article)
{
int bracketLength = 0;
return (Parsers.UnbalancedBrackets(article.Text, ref bracketLength) != -1);
}
public override bool Check(ArticleInfo article)
{
return(Retf.DetectTypo(article.Text, article.Title));
}
public override bool Check(ArticleInfo article)
{
return Retf.DetectTypo(article.Text, article.Title);
}
public override bool Check(ArticleInfo article)
{
Actual = Tools.WordCount(article.Text);
if (M == MoreLessThan.MoreThan)
return (Actual > Test);
if (M == MoreLessThan.LessThan)
return (Actual < Test);
return (Actual == Test);
}
public override bool Check(ArticleInfo article)
{
return (!Tools.IsRedirect(article.Text));
}
private void Process()
{
string articleTitle = "";
try
{
using (XmlTextReader reader = new XmlTextReader(stream))
{
reader.WhitespaceHandling = WhitespaceHandling.None;
if (From.Length > 0)
{
//move to start from article
while (reader.Read() && mRun)
{
if (reader.Name == "page")
{
reader.ReadToFollowing("title");
articleTitle = reader.ReadString();
if (From == articleTitle)
{
break;
}
}
}
}
while (reader.Read() && mRun)
{
if (reader.Name == "page")
{
ArticleInfo ai = new ArticleInfo();
reader.ReadToFollowing("title");
ai.Title = articleTitle = reader.ReadString();
//reader.ReadToFollowing(restriction); //TODO:This is wrong. Only want to read the restriction if in that <page></page>
if (reader.Name == "restrictions")
{
ai.Restrictions = reader.ReadString();
}
else
{
ai.Restrictions = "";
}
reader.ReadToFollowing("timestamp");
ai.Timestamp = reader.ReadString();
reader.ReadToFollowing("text");
ai.Text = reader.ReadString();
if (IgnoreComments)
{
ai.Text = WikiRegexes.Comments.Replace(ai.Text, "");
}
if (MultiThreaded)
{
if (PendingArticles.Count < ProcessorCount * 4 + 5)
{
PendingArticles.Add(ai);
}
else
{
ScanArticle(ai);
}
}
else
{
ScanArticle(ai);
}
}
}
lock (ScanThread)
{
stream = null;
}
if (MultiThreaded)
{
while (PendingArticles.Count > 0)
{
Thread.Sleep(10);
}
mRun = false;
foreach (Thread thr in SecondaryThreads)
{
thr.Join();
}
}
}
}
catch (ThreadAbortException) { }
catch (Exception ex)
{
if (boolMessage)
{
//System.Windows.Forms.MessageBox.Show("Problem on " + articleTitle + "\r\n\r\n" + ex.Message);
ErrorHandler.Handle(ex);
}
}
finally
{
if (boolMessage)
{
context.Post(SOPCstopped, articleTitle);
}
}
}
public override bool Check(ArticleInfo article)
{
DateTime timestamp;
if (DateTime.TryParse(article.Timestamp, out timestamp))
return ((DateTime.Compare(timestamp, From) >= 0) && (DateTime.Compare(timestamp, To) <= 0));
return false;
}
public override bool Check(ArticleInfo article)
{
Actual = article.Text.Length;
if (M == MoreLessThan.MoreThan)
return (Actual > Test);
if (M == MoreLessThan.LessThan)
return (Actual < Test);
return (Actual == Test);
}
public override bool Check(ArticleInfo article)
{
bool restrictionStringEmpty = (string.IsNullOrEmpty(article.Restrictions));
bool noEditRestriction = string.IsNullOrEmpty(Edit);
bool noMoveRestriction = string.IsNullOrEmpty(Move);
if (restrictionStringEmpty)
{
return (noEditRestriction && noMoveRestriction);
}
if (!noEditRestriction && !article.Restrictions.Contains(EditRest + Edit))
return false;
return noMoveRestriction || article.Restrictions.Contains(MoveRest + Move);
}
public override bool Check(ArticleInfo article)
{
P.FixPeopleCategories(article.Text, article.Title, false, out Skip);
return !Skip;
}
public override bool Check(ArticleInfo article)
{
if (WikiRegexes.Category.IsMatch(article.Text))
return false;
foreach (Match m in WikiRegexes.Template.Matches(article.Text))
{
if (!m.Value.Contains("stub"))
return false;
}
return true;
}
public override bool Check(ArticleInfo article)
{
P.Unicodify(article.Text, out Skip);
return(!Skip);
}
public override bool Check(ArticleInfo article)
{
return !base.Check(article);
}
public override bool Check(ArticleInfo article)
{
Parsers.FixHeadings(article.Text, article.Title, out Skip);
return(!Skip);
}
public override bool Check(ArticleInfo article)
{
P.CiteTemplateDates(article.Text, out Skip);
return !Skip;
}
public override bool Check(ArticleInfo article)
{
Parsers.LivingPeople(article.Text, out Skip);
return(!Skip);
}
private void ScanArticle(ArticleInfo ai)
{
foreach (Scan z in Scanners)
{
if (!z.Check(ref ai.Text, ref ai.Title, ai.Timestamp, ai.Restrictions))
{
return;
}
}
OutputQueue.Add(ai.Title);
}
public override bool Check(ArticleInfo article)
{
Parsers.ChangeToDefaultSort(article.Text, article.Title, out Skip);
return(!Skip);
}
public override bool Check(ArticleInfo article)
{
P.BoldTitle(article.Text, article.Title, out Skip);
return !Skip;
}
public override bool Check(ArticleInfo article)
{
return (Namespaces.Contains(Namespace.Determine(article.Title)));
}
public override bool Check(ArticleInfo article)
{
return (Contains.IsMatch(article.Title));
}
public override bool Check(ArticleInfo article)
{
Parsers.LivingPeople(article.Text, out Skip);
return !Skip;
}
public override bool Check(ArticleInfo article)
{
P.BoldTitle(article.Text, article.Title, out Skip);
return(!Skip);
}
private void Process()
{
string articleTitle = "";
try
{
using (XmlTextReader reader = new XmlTextReader(stream))
{
reader.WhitespaceHandling = WhitespaceHandling.None;
if (From.Length > 0)
{
//move to start from article
while (reader.Read() && mRun)
{
if (reader.Name == "page")
{
reader.ReadToFollowing("title");
articleTitle = reader.ReadString();
if (From == articleTitle)
break;
}
}
}
while (reader.Read() && mRun)
{
if (reader.Name == "page")
{
ArticleInfo ai = new ArticleInfo();
reader.ReadToFollowing("title");
ai.Title = articleTitle = reader.ReadString();
//reader.ReadToFollowing(restriction); //TODO:This is wrong. Only want to read the restriction if in that <page></page>
if (reader.Name == "restrictions")
ai.Restrictions = reader.ReadString();
else
ai.Restrictions = "";
reader.ReadToFollowing("timestamp");
ai.Timestamp = reader.ReadString();
reader.ReadToFollowing("text");
ai.Text = reader.ReadString();
if (IgnoreComments)
ai.Text = WikiRegexes.Comments.Replace(ai.Text, "");
if (MultiThreaded)
{
if (PendingArticles.Count < ProcessorCount * 4 + 5)
{
PendingArticles.Add(ai);
}
else
{
ScanArticle(ai);
}
}
else
{
ScanArticle(ai);
}
}
}
lock (ScanThread)
{
stream = null;
}
if (MultiThreaded)
{
while (PendingArticles.Count > 0)
Thread.Sleep(10);
mRun = false;
foreach (Thread thr in SecondaryThreads)
thr.Join();
}
}
}
catch (ThreadAbortException) { }
catch (Exception ex)
{
if (boolMessage)
//System.Windows.Forms.MessageBox.Show("Problem on " + articleTitle + "\r\n\r\n" + ex.Message);
ErrorHandler.Handle(ex);
}
finally
{
if (boolMessage)
context.Post(SOPCstopped, articleTitle);
}
}
public override bool Check(ArticleInfo article)
{
P.CiteTemplateDates(article.Text, out Skip);
return(!Skip);
}
public override bool Check(ArticleInfo article)
{
Actual = WikiRegexes.SimpleWikiLink.Matches(article.Text).Count;
if (M == MoreLessThan.MoreThan)
return (Actual > Test);
if (M == MoreLessThan.LessThan)
return (Actual < Test);
return (Actual == Test);
}
public override bool Check(ArticleInfo article)
{
string articleTextAfter = Parsers.ReorderReferences(article.Text);
return(!articleTextAfter.Equals(article.Text));
}
private void ScanArticle(ArticleInfo ai)
{
if (IgnoreComments)
ai.Text = WikiRegexes.Comments.Replace(ai.Text, "");
foreach (Scan z in Scanners)
{
if (!z.Check(ai))
{
return;
}
}
OutputQueue.Add(ai.Title);
}
public virtual bool Check(ArticleInfo article)
{
return(true);
}
private void Process()
{
string articleTitle = "";
try
{
using (XmlTextReader reader = new XmlTextReader(stream))
{
reader.WhitespaceHandling = WhitespaceHandling.None;
if (From.Length > 0)
{
//move to start from article
while (Run && reader.Read())
{
if (reader.NodeType != XmlNodeType.Element)
{
continue;
}
if (reader.Name != "title")
{
reader.ReadToFollowing("page");
continue;
}
//reader.ReadToFollowing("title");
articleTitle = reader.ReadString();
if (From == articleTitle)
{
break;
}
}
}
while (Run)
{
ArticleInfo ai = ReadArticle(reader);
if (ai == null)
{
break;
}
articleTitle = ai.Title;
// we must maintain a huge enough buffer to safeguard against fluctuations
// of page size
if (MultiThreaded && (PendingArticles.Count < ProcessorCount * 10))
{
PendingArticles.Add(ai);
}
else
{
ScanArticle(ai);
}
}
lock (ScanThread)
{
stream = null;
}
if (MultiThreaded)
{
while (PendingArticles.Count > 0)
{
Thread.Sleep(10);
}
Run = false;
foreach (Thread thr in SecondaryThreads)
{
thr.Join();
}
}
}
}
catch (ThreadAbortException) { }
catch (Exception ex)
{
if (Message)
{
//System.Windows.Forms.MessageBox.Show("Problem on " + articleTitle + "\r\n\r\n" + ex.Message);
ErrorHandler.Handle(ex);
}
}
finally
{
if (Message)
{
Context.Post(SOPCstopped, articleTitle);
}
}
}
public override bool Check(ArticleInfo article)
{
P.FixPeopleCategories(article.Text, article.Title, false, out Skip);
return(!Skip);
}
public override bool Check(ArticleInfo article)
{
return Parsers.MissingDefaultSort(article.Text, article.Title);
}
public override bool Check(ArticleInfo article)
{
string articleTextAfter = Parsers.ReorderReferences(article.Text);
return !articleTextAfter.Equals(article.Text);
}
