private static List<WikiPageSection> SubsectionsList(WikiPageSection section,
List<WikiPageSection> aggregator)
{
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
Match m = wikiLinkRE.Match(section.Title);
if (m.Success)
{
aggregator.Add(section);
}
return section.Reduce(aggregator, SubsectionsList);
}
private string SubsectionsList(WikiPageSection section,
string aggregator)
{
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
Match m = wikiLinkRE.Match(section.Title);
if (m.Success)
{
if (_l10i.Processor != null)
{
aggregator = aggregator + " • " + _l10i.Processor(section).Trim();
}
else
{
aggregator = aggregator + " • " + section.Title.Trim();
}
}
aggregator = section.Reduce(aggregator, SubsectionsList);
return aggregator;
}
public void UpdatePages(Wiki wiki)
{
Regex closedRE = new Regex(@"(\{{2}ВПКПМ-Навигация\}{2}\s*\{{2}(Закрыто|Closed|закрыто|closed)\}{2})|(\{{2}(Закрыто|Closed|закрыто|closed)\}{2}\s*\{{2}ВПКПМ-Навигация\}{2})");
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
Regex timeRE = new Regex(@"(\d{2}:\d{2}\, \d\d? [а-я]+ \d{4}) \(UTC\)");
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения переименования страниц");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info|revisions");
parameters.Add("intoken", "edit");
XmlDocument doc = wiki.Enumerate(parameters, true);
string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
XmlNodeList pages = doc.SelectNodes("//page");
foreach (XmlNode page in pages)
{
int results = 0;
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring("Википедия:К переименованию/".Length);
string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value;
string editToken = page.Attributes["edittoken"].Value;
Day day = new Day();
if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
string fileName = _cacheDir + date + ".bin";
string text = "";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
day.Page = WikiPage.Parse(pageName, text);
List<string> titlesWithResults = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
RemoveStrikeOut(section);
StrikeOutSection(section);
Match m = wikiLinkRE.Match(section.Title);
if (m.Success && !m.Groups[1].Value.StartsWith(":Категория:"))
{
string link = m.Groups[1].Value;
string movedTo;
string movedBy;
DateTime movedAt;
DateTime start = day.Date;
m = timeRE.Match(section.SectionText);
if (m.Success)
{
start = DateTime.Parse(m.Groups[1].Value,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
bool moved = MovedTo(wiki,
link,
start,
out movedTo,
out movedBy,
out movedAt);
WikiPageSection autoVerdictSection = section.Subsections.FirstOrDefault(s => s.Title.Trim() == "Автоматический итог");
if (autoVerdictSection != null && !moved)
{
autoVerdictSection.Title = " Оспоренный итог ";
RemoveStrikeOut(section);
StrikeOutSection(section);
}
if (section.Subsections.Count(s => s.Title.Trim() == "Оспоренный итог") == 0 &&
section.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") == 0 &&
section.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") == 0 &&
moved && !string.IsNullOrEmpty(movedTo))
{
string message = string.Format("Страница была переименована {2} в «[[{0}]]» участником [[User:{1}|]]. Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
movedTo,
movedBy,
movedAt.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)"));
WikiPageSection verdict = new WikiPageSection(" Автоматический итог ",
section.Level + 1,
message);
section.AddSubsection(verdict);
StrikeOutSection(section);
++results;
}
}
m = wikiLinkRE.Match(section.Title);
if (m.Success && section.Title.Contains("<s>"))
{
titlesWithResults.Add(m.Groups[1].Value);
}
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
m = wikiLinkRE.Match(subsection.Title);
if (m.Success && subsection.Title.Contains("<s>"))
{
titlesWithResults.Add(m.Groups[1].Value.Trim());
}
if (m.Success &&
!subsection.Title.Contains("<s>") &&
!m.Groups[1].Value.StartsWith(":Категория:"))
{
string link = m.Groups[1].Value;
string movedTo;
string movedBy;
DateTime movedAt;
DateTime start = day.Date;
m = timeRE.Match(subsection.SectionText);
if (m.Success)
{
start = DateTime.Parse(m.Groups[1].Value,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
bool moved = MovedTo(wiki,
link,
start,
out movedTo,
out movedBy,
out movedAt);
WikiPageSection verdictSection = subsection.Subsections.FirstOrDefault(s => s.Title.Trim() == "Автоматический итог");
if (verdictSection != null && !moved)
{
verdictSection.Title = " Оспоренный итог ";
RemoveStrikeOut(subsection);
StrikeOutSection(subsection);
}
if (subsection.Subsections.Count(s => s.Title.Trim() == "Оспоренный итог") == 0 &&
subsection.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") == 0 &&
subsection.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") == 0 &&
moved && !string.IsNullOrEmpty(movedTo))
{
string message = string.Format("Страница была переименована {2} в «[[{0}]]» участником [[User:{1}|]]. Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
movedTo,
movedBy,
movedAt.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)"));
WikiPageSection verdict = new WikiPageSection(" Автоматический итог ",
subsection.Level + 1,
message);
subsection.AddSubsection(verdict);
StrikeOutSection(subsection);
++results;
}
}
}
}
Match matchClosed = closedRE.Match(text);
if (matchClosed.Success)
{
List<TalkResult> talkResults = new List<TalkResult>();
foreach (string name in titlesWithResults)
{
if (wiki.PageNamespace(name) > 0)
{
continue;
}
string movedTo;
string movedBy;
DateTime movedAt;
bool moved = MovedTo(wiki,
name,
day.Date,
out movedTo,
out movedBy,
out movedAt);
talkResults.Add(new TalkResult(name, movedTo, moved));
}
parameters.Clear();
parameters.Add("prop", "info");
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, talkResults.ConvertAll(r => r.Moved ? r.MovedTo : r.Title));
List<string> notificationList = new List<string>();
foreach (XmlNode node in xml.SelectNodes("//page"))
{
if (node.Attributes["missing"] == null && node.Attributes["ns"].Value == "0")
{
notificationList.Add(node.Attributes["title"].Value);
}
}
if (notificationList.Count > 0)
{
parameters.Clear();
parameters.Add("list", "backlinks");
parameters.Add("bltitle", pageName);
parameters.Add("blfilterredir", "nonredirects");
parameters.Add("blnamespace", "1");
parameters.Add("bllimit", "max");
XmlDocument backlinks = wiki.Enumerate(parameters, true);
foreach (string title in notificationList)
{
if (backlinks.SelectSingleNode("//bl[@title=\"Обсуждение:" + title + "\"]") == null)
{
TalkResult tr = talkResults.Find(r => r.Moved ? r.MovedTo == title : r.Title == title);
PutNotification(wiki, tr, day.Date);
}
}
}
}
string newText = day.Page.Text;
if (newText.Trim() == text.Trim())
{
continue;
}
try
{
Console.Out.WriteLine("Updating " + pageName + "...");
string revid = wiki.Save(pageName,
"",
newText,
"зачёркивание заголовков" + (results > 0 ? ", сообщение об итогах" : ""),
MinorFlags.Minor,
CreateFlags.NoCreate,
WatchFlags.None,
SaveFlags.Replace,
true,
basetimestamp,
"",
editToken);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(revid);
sw.Write(newText);
}
}
catch (WikiException)
{
}
}
}
private void ReplaceEmptyResults(WikiPageSection section)
{
WikiPageSection result = section.Subsections.FirstOrDefault(s => _l10i.Processor != null
? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim())
: _l10i.Results.Any(r => r == s.Title.Trim()));
if (result != null && result.Subsections.Count == 0 &&
string.IsNullOrEmpty(result.SectionText.Trim()))
{
result.Title = _l10i.EmptyResult;
}
section.ForEach(ReplaceEmptyResults);
}
private void StrikeOutSection(WikiPageSection section)
{
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
if (section.Subsections.Count(s => _l10i.Processor != null
? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim())
: _l10i.Results.Any(r => r == s.Title.Trim())) > 0)
{
if (!section.Title.Contains("<s>"))
{
section.Title = string.Format(" <s>{0}</s> ",
section.Title.Trim());
}
foreach (WikiPageSection subsection in section.Subsections)
{
Match m = wikiLinkRE.Match(subsection.Title);
if (m.Success && !subsection.Title.Contains("<s>"))
{
subsection.Title = string.Format(" <s>{0}</s> ",
subsection.Title.Trim());
}
}
}
section.ForEach(StrikeOutSection);
}
private void RemoveStrikeOut(WikiPageSection section)
{
if (section.Subsections.Count(s => s.Title.Trim() == "Итог") == 0)
{
if (section.Title.Contains("<s>"))
{
section.Title = section.Title.Replace("<s>", "");
section.Title = section.Title.Replace("</s>", "");
}
}
section.ForEach(RemoveStrikeOut);
}
static string RemoveVotes(WikiPageSection section)
{
Regex re = new Regex(@"\s+\d{1,3}—\d{1,3}(—\d{1,3})?\s*(</s>)?\s*$");
return re.Replace(section.Title, "$2");
}
public void Parse(string text)
{
if (_sections.Count != 0)
{
_sections.Clear();
}
if (!string.IsNullOrEmpty(_text))
{
_text = "";
}
StringReader reader = new StringReader(text);
StringBuilder sectionText = new StringBuilder();
int level = 0;
string sectionTitle = "";
string rawSectionTitle = "";
bool found = false;
Tokens token = Tokens.WikiText;
string line;
while ((line = reader.ReadLine()) != null)
{
for (int i = 0; i < line.Length - 1; ++i)
{
char ch = line[i];
char nextCh = line[i + 1];
switch (token)
{
case Tokens.WikiText:
if (ch == '<' && nextCh == '!')
{
token = Tokens.HtmlCommentStart;
++i;
}
else if (ch == '<' && nextCh == 'n')
{
token = Tokens.NoWikiStartPart1;
++i;
}
break;
case Tokens.HtmlCommentStart:
if (ch == '-' && nextCh == '-')
{
token = Tokens.HtmlComment;
++i;
}
else
{
token = Tokens.WikiText;
}
break;
case Tokens.HtmlComment:
if (ch == '-' && nextCh == '-')
{
token = Tokens.HtmlCommentEnd;
}
break;
case Tokens.HtmlCommentEnd:
if (ch == '-' && nextCh == '>')
{
token = Tokens.WikiText;
}
else if (ch == '-' && nextCh == '-')
{
break;
}
else
{
token = Tokens.HtmlComment;
}
break;
case Tokens.NoWikiStartPart1:
if (ch == 'o' && nextCh == 'w')
{
token = Tokens.NoWikiStartPart2;
++i;
}
else
{
token = Tokens.WikiText;
}
break;
case Tokens.NoWikiStartPart2:
if (ch == 'i' && nextCh == 'k')
{
token = Tokens.NoWikiStartPart3;
++i;
}
else
{
token = Tokens.WikiText;
}
break;
case Tokens.NoWikiStartPart3:
if (ch == 'i' && nextCh == '>')
{
token = Tokens.NoWiki;
++i;
}
else
{
token = Tokens.WikiText;
}
break;
case Tokens.NoWiki:
if (ch == '<' && nextCh == '/')
{
token = Tokens.NoWikiEndPart1;
++i;
}
break;
case Tokens.NoWikiEndPart1:
if (ch == 'n' && nextCh == 'o')
{
token = Tokens.NoWikiEndPart2;
++i;
}
else
{
token = Tokens.NoWiki;
}
break;
case Tokens.NoWikiEndPart2:
if (ch == 'w' && nextCh == 'i')
{
token = Tokens.NoWikiEndPart3;
++i;
}
else
{
token = Tokens.NoWiki;
}
break;
case Tokens.NoWikiEndPart3:
if (ch == 'k' && nextCh == 'i')
{
token = Tokens.NoWikiEndPart4;
}
else
{
token = Tokens.NoWiki;
}
break;
case Tokens.NoWikiEndPart4:
if (ch == 'i' && nextCh == '>')
{
token = Tokens.WikiText;
}
else
{
token = Tokens.NoWiki;
}
break;
default:
break;
}
}
if (token == Tokens.HtmlCommentStart)
{
token = Tokens.WikiText;
}
else if (token == Tokens.HtmlCommentEnd)
{
token = Tokens.HtmlComment;
}
else if (token == Tokens.NoWikiStartPart1 ||
token == Tokens.NoWikiStartPart2 ||
token == Tokens.NoWikiStartPart3)
{
token = Tokens.WikiText;
}
else if (token == Tokens.NoWikiEndPart1 ||
token == Tokens.NoWikiEndPart2 ||
token == Tokens.NoWikiEndPart3 ||
token == Tokens.NoWikiEndPart4)
{
token = Tokens.NoWiki;
}
Match m = _sectionRE.Match(line);
if (token == Tokens.WikiText && m.Success)
{
if (found)
{
var section = new WikiPageSection(sectionTitle,
rawSectionTitle,
level,
sectionText.ToString());
int index = _sections.Count - 1;
if (index >= 0 &&
index < _sections.Count &&
_sections[index].Level < level)
{
_sections[index].AddSubsection(section);
}
else
{
_sections.Add(section);
}
sectionText = new StringBuilder();
found = false;
}
else
{
_text = sectionText.ToString();
sectionText = new StringBuilder();
}
found = true;
level = Math.Min(m.Groups[1].Length, m.Groups[3].Length);
sectionTitle = m.Groups[2].Value;
rawSectionTitle = line;
}
else
{
sectionText.Append(line + "\n");
}
}
if (found)
{
var section = new WikiPageSection(sectionTitle,
rawSectionTitle,
level,
sectionText.ToString());
int index = _sections.Count - 1;
if (index >= 0 &&
index < _sections.Count &&
_sections[index].Level < level)
{
_sections[index].AddSubsection(section);
}
else
{
_sections.Add(section);
}
}
else
{
_text = sectionText.ToString();
}
}
public void Run(Wiki wiki)
{
Directory.CreateDirectory(_cacheDir);
Console.Out.WriteLine("Updating " + _page);
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
Regex timeRE = new Regex(@"(\d{2}:\d{2}\, \d\d? [а-я]+ \d{4}) \(UTC\)");
ParameterCollection parameters = new ParameterCollection();
parameters.Add("prop", "info|revisions");
parameters.Add("intoken", "edit");
XmlDocument doc = wiki.Query(QueryBy.Titles, parameters, _page);
XmlNode page = doc.SelectSingleNode("//page");
string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
string pageName = page.Attributes["title"].Value;
string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value;
string editToken = page.Attributes["edittoken"].Value;
string starttimestamp = queryTimestamp;
string text = "";
string fileName = _cacheDir + WikiCache.EscapePath(_page);
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
starttimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
Dictionary<string, List<WikiPageSection>> titles = new Dictionary<string, List<WikiPageSection>>();
WikiPage wikiPage = WikiPage.Parse(pageName, text);
foreach (WikiPageSection section in wikiPage.Sections)
{
if (section.Subsections.Count(s => s.Title.ToLower(CultureInfo.CreateSpecificCulture("ru-RU")).Trim() == "итог") == 0)
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(section);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(section);
}
}
}
}
parameters.Clear();
parameters.Add("prop", "info");
Dictionary<string, string> normalizedTitles = new Dictionary<string, string>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles.Keys);
foreach (XmlNode node in xml.SelectNodes("//n"))
{
normalizedTitles.Add(node.Attributes["to"].Value,
node.Attributes["from"].Value);
}
List<string> notificationList = new List<string>();
XmlNodeList missingTitles = xml.SelectNodes("//page");
foreach (XmlNode node in missingTitles)
{
string title = node.Attributes["title"].Value;
IEnumerable<WikiPageSection> sections;
if (titles.ContainsKey(title))
{
sections = titles[title];
}
else
{
sections = titles[normalizedTitles[title]];
}
if (node.Attributes["missing"] != null)
{
parameters.Clear();
parameters.Add("list", "logevents");
parameters.Add("letype", "delete");
parameters.Add("lemlimit", "max");
parameters.Add("ledir", "newer");
parameters.Add("letitle", title);
parameters.Add("leprop", "comment|type|user|timestamp");
XmlDocument log = wiki.Enumerate(parameters, true);
XmlNodeList items = log.SelectNodes("//item");
List<DeleteLogEvent> events = new List<DeleteLogEvent>();
foreach (XmlNode item in items)
{
DeleteLogEvent ev = new DeleteLogEvent();
ev.Comment = item.Attributes["comment"].Value;
ev.Deleted = item.Attributes["action"].Value == "delete";
ev.User = item.Attributes["user"].Value;
ev.Timestamp = DateTime.Parse(item.Attributes["timestamp"].Value,
null,
DateTimeStyles.AssumeUniversal);
events.Add(ev);
}
events.Sort(CompareDeleteLogEvents);
if (events.Count > 0 && events[0].Deleted)
{
string comment = FilterWikiMarkup(events[0].Comment);
string message = string.Format("Страница была удалена {1} администратором [[User:{0}|]]. Была указана следующая причина: «{2}». Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
events[0].User,
events[0].Timestamp.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)",
CultureInfo.CreateSpecificCulture("ru-RU")),
comment);
var pageSections = titles.ContainsKey(title)
? titles[title]
: titles[normalizedTitles[title]];
foreach (WikiPageSection section in pageSections)
{
WikiPageSection verdict = new WikiPageSection(" Итог ",
section.Level + 1,
message);
section.AddSubsection(verdict);
}
}
}
}
string newText = wikiPage.Text;
if (newText.Trim() == text.Trim())
{
return;
}
Console.Out.WriteLine("Updating " + pageName + "...");
string id = wiki.Save(pageName,
"",
newText,
"автоматическое подведение итогов",
MinorFlags.Minor,
CreateFlags.NoCreate,
WatchFlags.None,
SaveFlags.Replace,
true,
basetimestamp,
"",
editToken);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(id);
sw.Write(newText);
}
}
internal void UpdatePages(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения восстановления страниц");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info|revisions");
parameters.Add("intoken", "edit");
XmlDocument doc = wiki.Enumerate(parameters, true);
string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
XmlNodeList pages = doc.SelectNodes("//page");
foreach (XmlNode page in pages)
{
string starttimestamp = queryTimestamp;
int results = 0;
string prefix = "Википедия:К восстановлению/";
string pageName = page.Attributes["title"].Value;
string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value;
string editToken = page.Attributes["edittoken"].Value;
string date = pageName.Substring(prefix.Length);
Day day = new Day();
if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
string text = "";
string fileName = _cacheDir + date + ".bin";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
try
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
starttimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
}
catch (WikiPageNotFound)
{
continue;
}
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
Dictionary<string, List<WikiPageSection>> titles = new Dictionary<string, List<WikiPageSection>>();
day.Page = WikiPage.Parse(pageName, text);
foreach (WikiPageSection section in day.Page.Sections)
{
RemoveStrikeOut(section);
StrikeOutSection(section);
if (section.Subsections.Count(s => _results.Any(r => r.ToLower() == s.Title.Trim().ToLower())) == 0)
{
Match m = _wikiLinkRE.Match(section.Title);
if (m.Success)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(section);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(section);
}
}
}
{
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
Match m = _wikiLinkRE.Match(subsection.Title);
if (m.Success &&
!subsection.Title.Contains("<s>") &&
subsection.Subsections.Count(s => _results.Any(r => r.ToLower() == s.Title.Trim().ToLower())) == 0)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(subsection);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(subsection);
}
}
}
}
}
parameters.Clear();
parameters.Add("prop", "info");
Dictionary<string, string> normalizedTitles = new Dictionary<string, string>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles.Keys);
foreach (XmlNode node in xml.SelectNodes("//n"))
{
normalizedTitles.Add(node.Attributes["to"].Value,
node.Attributes["from"].Value);
}
List<string> notificationList = new List<string>();
XmlNodeList missingTitles = xml.SelectNodes("//page");
foreach (XmlNode node in missingTitles)
{
string title = node.Attributes["title"].Value;
IEnumerable<WikiPageSection> sections;
if (titles.ContainsKey(title))
{
sections = titles[title];
}
else
{
sections = titles[normalizedTitles[title]];
}
if (node.Attributes["missing"] == null)
{
DateTime start = day.Date;
parameters.Clear();
parameters.Add("list", "logevents");
parameters.Add("letype", "delete");
parameters.Add("lemlimit", "max");
parameters.Add("lestart", start.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"));
parameters.Add("ledir", "newer");
parameters.Add("letitle", title);
parameters.Add("leprop", "comment|type|user|timestamp");
XmlDocument log = wiki.Enumerate(parameters, true);
XmlNodeList items = log.SelectNodes("//item");
List<DeleteLogEvent> events = new List<DeleteLogEvent>();
foreach (XmlNode item in items)
{
DeleteLogEvent ev = new DeleteLogEvent();
ev.Comment = item.Attributes["comment"].Value;
ev.Restored = item.Attributes["action"].Value == "restore";
ev.Deleted = item.Attributes["action"].Value == "delete";
ev.User = item.Attributes["user"].Value;
ev.Timestamp = DateTime.Parse(item.Attributes["timestamp"].Value,
null,
DateTimeStyles.AssumeUniversal);
events.Add(ev);
}
events.Sort(CompareDeleteLogEvents);
if (events.Count > 0 &&
events[0].Restored &&
(DateTime.Now - events[0].Timestamp).TotalHours > 2)
{
Regex commentRE = new Regex(@" восстановлено: (.+)");
Match m = commentRE.Match(events[0].Comment);
string comment;
if (m.Success)
{
comment = m.Groups[1].Value;
}
else
{
comment = "<nowiki>" + events[0].Comment + "</nowiki>";
}
string message = string.Format("Страница была восстановлена {1} администратором [[User:{0}|]]. Была указана следующая причина: «{2}». Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
events[0].User,
events[0].Timestamp.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)"),
comment);
if (!titles.ContainsKey(title))
{
continue;
}
foreach (WikiPageSection section in titles[title])
{
WikiPageSection verdict = new WikiPageSection(" Автоматический итог ",
section.Level + 1,
message);
section.AddSubsection(verdict);
StrikeOutSection(section);
++results;
}
}
else if (events.Count > 0 &&
events[0].Deleted)
{
parameters.Clear();
parameters.Add("prop", "revisions");
parameters.Add("rvprop", "timestamp|user");
parameters.Add("rvdir", "newer");
parameters.Add("rvlimit", "1");
parameters.Add("redirects");
log = wiki.Query(QueryBy.Titles, parameters, new string[] { title });
XmlNode revision = log.SelectSingleNode("//rev");
if (revision != null)
{
string user = revision.Attributes["user"].Value;
string timestamp = revision.Attributes["timestamp"].Value;
DateTime time = DateTime.Parse(timestamp,
null,
DateTimeStyles.AssumeUniversal);
string message = string.Format("Страница была создана заново {1} участником [[User:{0}|]]. Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
user,
time.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)"));
if (!titles.ContainsKey(title))
{
continue;
}
foreach (WikiPageSection section in titles[title])
{
WikiPageSection verdict = new WikiPageSection(" Автоматический итог ",
section.Level + 1,
message);
section.AddSubsection(verdict);
StrikeOutSection(section);
++results;
}
}
}
}
}
string newText = day.Page.Text;
if (newText.Trim() == text.Trim())
{
continue;
}
try
{
Console.Out.WriteLine("Updating " + pageName + "...");
string revid = wiki.Save(pageName,
"",
newText,
"зачёркивание заголовков" + (results > 0 ? " и подведение итогов" : ""),
MinorFlags.Minor,
CreateFlags.NoCreate,
WatchFlags.None,
SaveFlags.Replace,
true,
basetimestamp,
"",
editToken);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(revid);
sw.Write(newText);
}
}
catch (WikiException)
{
}
}
}
private void RemoveStrikeOut(WikiPageSection section)
{
if (section.Subsections.Count(s => _results.Any(r => r.ToLower() == s.Title.Trim().ToLower())) == 0)
{
if (section.Title.Contains("<s>"))
{
section.Title = section.Title.Replace("<s>", "");
section.Title = section.Title.Replace("</s>", "");
}
}
section.ForEach(RemoveStrikeOut);
}
protected int SectionsUp(WikiPageSection x, WikiPageSection y)
{
MatchCollection ms = timeRE.Matches(x.Text);
DateTime publishedX = DateTime.MaxValue;
foreach (Match match in ms)
{
string value = match.Groups[1].Value;
DateTime time;
if (DateTime.TryParse(value, L10i.Culture,
DateTimeStyles.AssumeUniversal, out time) &&
time < publishedX)
{
publishedX = time;
}
}
if (publishedX == DateTime.MaxValue)
{
publishedX = DateTime.MinValue;
}
ms = timeRE.Matches(y.Text);
DateTime publishedY = DateTime.MaxValue;
foreach (Match match in ms)
{
string value = match.Groups[1].Value;
DateTime time;
if (DateTime.TryParse(value, L10i.Culture,
DateTimeStyles.AssumeUniversal, out time) &&
time < publishedY)
{
publishedY = time;
}
}
if (publishedY == DateTime.MaxValue)
{
publishedY = DateTime.MinValue;
}
return publishedY.CompareTo(publishedX);
}
protected static bool IsMovedSection(WikiPageSection section)
{
Regex re = new Regex(@"^\s*\{\{(moved|перенесено на|обсуждение перенесено|moved to|перенесено в)\|(.+?)\}\}\s*$", RegexOptions.IgnoreCase);
Match m = re.Match(section.SectionText);
return m.Success;
}
private void RemoveStrikeOut(WikiPageSection section)
{
if (!section.Title.ToLower(_l10i.Culture).Contains("{{ok}}") &&
!section.Title.ToLower(_l10i.Culture).Contains("{{ок}}") &&
!section.Title.ToLower(_l10i.Culture).Contains("{{x}}") &&
section.Subsections.Count(s => s.Title.Trim() == "Итог") == 0)
{
if (section.Title.Contains("<s>"))
{
section.Title = section.Title.Replace("<s>", "");
section.Title = section.Title.Replace("</s>", "");
}
}
section.ForEach(RemoveStrikeOut);
}
private void StrikeOutSection(WikiPageSection section)
{
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
if (section.Subsections.Count(s => s.Title.ToLower().Trim() == "итог" ||
s.Title.Trim() == "Автоматический итог") > 0)
{
if (!section.Title.Contains("<s>"))
{
section.Title = string.Format(" <s>{0}</s> ",
section.Title.Trim());
}
foreach (WikiPageSection subsection in section.Subsections)
{
Match m = wikiLinkRE.Match(subsection.Title);
if (m.Success && !subsection.Title.Contains("<s>"))
{
subsection.Title = string.Format(" <s>{0}</s> ",
subsection.Title.Trim());
}
}
}
section.ForEach(StrikeOutSection);
}
public void UpdatePages(Wiki wiki)
{
Console.Out.WriteLine("Updating articles for deletion...");
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
Regex timeRE = new Regex(@"(\d{2}:\d{2}\, \d\d? [а-я]+ \d{4}) \(UTC\)");
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", _l10i.Category);
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info|revisions");
parameters.Add("intoken", "edit");
XmlDocument doc = wiki.Enumerate(parameters, true);
string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
XmlNodeList pages = doc.SelectNodes("//page");
foreach (XmlNode page in pages)
{
string starttimestamp = queryTimestamp;
int results = 0;
string prefix = _l10i.MainPage + "/";
string pageName = page.Attributes["title"].Value;
string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value;
string editToken = page.Attributes["edittoken"].Value;
if (pageName.Length < prefix.Length)
{
continue;
}
string date = pageName.Substring(prefix.Length);
Day day = new Day();
if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture(_l10i.Culture),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
string text = "";
string fileName = _cacheDir + date + ".bin";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
try
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
starttimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
}
catch (WikiPageNotFound)
{
continue;
}
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
List<string> titlesWithResults = new List<string>();
Dictionary<string, List<WikiPageSection>> titles = new Dictionary<string, List<WikiPageSection>>();
day.Page = WikiPage.Parse(pageName, text);
foreach (WikiPageSection section in day.Page.Sections)
{
ReplaceEmptyResults(section);
RemoveStrikeOut(section);
StrikeOutSection(section);
if (section.Subsections.Count(s => _l10i.Processor != null
? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim())
: _l10i.Results.Any(r => r == s.Title.Trim())) == 0 &&
section.Subsections.Count(s => s.Title.Trim() == _l10i.ChallengedResult) == 0)
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(section);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(section);
}
}
}
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success && section.Title.Contains("<s>"))
{
titlesWithResults.Add(m.Groups[1].Value.Trim());
}
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
m = wikiLinkRE.Match(subsection.Title);
if (m.Success && subsection.Title.Contains("<s>"))
{
titlesWithResults.Add(m.Groups[1].Value.Trim());
}
if (m.Success &&
!subsection.Title.Contains("<s>") &&
subsection.Subsections.Count(s => s.Title.Trim() == _l10i.ChallengedResult) == 0 &&
subsection.Subsections.Count(s => _l10i.Processor != null
? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim())
: _l10i.Results.Any(r => r == s.Title.Trim())) == 0)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(subsection);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(subsection);
}
}
}
}
}
parameters.Clear();
parameters.Add("prop", "info");
Dictionary<string, string> normalizedTitles = new Dictionary<string, string>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles.Keys);
foreach (XmlNode node in xml.SelectNodes("//n"))
{
normalizedTitles.Add(node.Attributes["to"].Value,
node.Attributes["from"].Value);
}
List<string> notificationList = new List<string>();
XmlNodeList missingTitles = xml.SelectNodes("//page");
foreach (XmlNode node in missingTitles)
{
string title = node.Attributes["title"].Value;
IEnumerable<WikiPageSection> sections;
if (titles.ContainsKey(title))
{
sections = titles[title];
}
else
{
sections = titles[normalizedTitles[title]];
}
if (node.Attributes["missing"] != null)
{
DateTime start = day.Date;
//foreach (WikiPageSection section in sections)
//{
// Match m = timeRE.Match(section.Text);
// if (m.Success)
// {
// start = DateTime.Parse(m.Groups[1].Value,
// CultureInfo.CreateSpecificCulture(_l10i.Culture),
// DateTimeStyles.AssumeUniversal);
// }
//}
parameters.Clear();
parameters.Add("list", "logevents");
parameters.Add("letype", "delete");
parameters.Add("lemlimit", "max");
parameters.Add("lestart", start.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"));
parameters.Add("ledir", "newer");
parameters.Add("letitle", title);
parameters.Add("leprop", "comment|type|user|timestamp");
XmlDocument log = wiki.Enumerate(parameters, true);
XmlNodeList items = log.SelectNodes("//item");
List<DeleteLogEvent> events = new List<DeleteLogEvent>();
foreach (XmlNode item in items)
{
DeleteLogEvent ev = new DeleteLogEvent();
ev.Comment = item.Attributes["comment"].Value;
ev.Deleted = item.Attributes["action"].Value == "delete";
ev.User = item.Attributes["user"].Value;
ev.Timestamp = DateTime.Parse(item.Attributes["timestamp"].Value,
null,
DateTimeStyles.AssumeUniversal);
events.Add(ev);
}
events.Sort(CompareDeleteLogEvents);
if (events.Count > 0 &&
events[0].Deleted &&
(DateTime.Now - events[0].Timestamp).TotalHours > 2)
{
string comment = FilterWikiMarkup(events[0].Comment);
string message = string.Format(_l10i.AutoResultMessage,
events[0].User,
events[0].Timestamp.ToUniversalTime().ToString(_l10i.DateFormat, CultureInfo.CreateSpecificCulture(_l10i.Culture)),
comment);
var pageSections = titles.ContainsKey(title)
? titles[title]
: titles[normalizedTitles[title]];
foreach (WikiPageSection section in pageSections)
{
WikiPageSection verdict = new WikiPageSection(" " + _l10i.AutoResultSection + " ",
section.Level + 1,
message);
section.AddSubsection(verdict);
StrikeOutSection(section);
++results;
}
}
}
}
if (_l10i.Culture != "ru-RU")
{
parameters.Clear();
parameters.Add("prop", "info");
xml = wiki.Query(QueryBy.Titles, parameters, titlesWithResults);
foreach (XmlNode node in xml.SelectNodes("//page"))
{
if (node.Attributes["missing"] == null &&
node.Attributes["redirect"] == null &&
node.Attributes["ns"].Value == "0")
{
notificationList.Add(node.Attributes["title"].Value);
}
}
if (notificationList.Count > 0)
{
parameters.Clear();
parameters.Add("list", "backlinks");
parameters.Add("bltitle", pageName);
parameters.Add("blfilterredir", "nonredirects");
parameters.Add("blnamespace", "1");
parameters.Add("bllimit", "max");
XmlDocument backlinks = wiki.Enumerate(parameters, true);
foreach (string title in notificationList)
{
string talkPage = wiki.GetNamespace(1) + ":" + title;
if (backlinks.SelectSingleNode("//bl[@title=" + GenerateConcatForXPath(talkPage) + "]") == null)
{
PutNotification(wiki, title, date);
}
}
}
}
string newText = day.Page.Text;
if (newText.Trim() == text.Trim())
{
continue;
}
try
{
Console.Out.WriteLine("Updating " + pageName + "...");
string revid = wiki.Save(pageName,
"",
newText,
_l10i.StrikeOutComment + (results > 0 ? _l10i.AutoResultComment : ""),
MinorFlags.Minor,
CreateFlags.NoCreate,
WatchFlags.None,
SaveFlags.Replace,
true,
basetimestamp,
"",
editToken);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(revid);
sw.Write(newText);
}
}
catch (WikiException)
{
}
}
}
private static List<WikiPageSection> SubsectionsList(WikiPageSection section,
List<WikiPageSection> aggregator)
{
Match m = _wikiLinkRE.Match(section.Title);
if (m.Success)
{
aggregator.Add(section);
}
return section.Reduce(aggregator, SubsectionsList);
}
private void RemoveStrikeOut(WikiPageSection section)
{
if (section.Subsections.Count(s => _l10i.Processor != null
? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim())
: _l10i.Results.Any(r => r == s.Title.Trim())) == 0)
{
if (section.Title.Contains("<s>"))
{
section.Title = section.Title.Replace("<s>", "");
section.Title = section.Title.Replace("</s>", "");
}
}
section.ForEach(RemoveStrikeOut);
}
private void StrikeOutSection(WikiPageSection section)
{
if (section.Subsections.Count(s => s.Title.Trim() == "Итог") > 0)
{
if (!section.Title.Contains("<s>"))
{
section.Title = string.Format(" <s>{0}</s> ",
section.Title.Trim());
}
foreach (WikiPageSection subsection in section.Subsections)
{
Match m = _wikiLinkRE.Match(subsection.Title);
if (m.Success && !subsection.Title.Contains("<s>"))
{
subsection.Title = string.Format(" <s>{0}</s> ",
subsection.Title.Trim());
}
}
}
section.ForEach(StrikeOutSection);
}
static string RemoveOK(WikiPageSection section)
{
Regex re = new Regex(@"^\s*(<s>)?\s*{{(ok|OK|Ok|oK|ОК|ок|Ок|оК|x|X)}}\s*(.+?)(</s>)?\s*$");
return re.Replace(section.Title, "$1$3</s>");
}
