public static WikiPage Load(Wiki wiki, string title, string directory)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("prop", "info|revisions");
parameters.Add("intoken", "edit");
parameters.Add("rvprop", "timestamp");
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, new string[] { title });
XmlNode node = xml.SelectSingleNode("//rev");
string baseTimeStamp = null;
if (node != null && node.Attributes["timestamp"] != null)
{
baseTimeStamp = node.Attributes["timestamp"].Value;
}
node = xml.SelectSingleNode("//page");
string editToken = node.Attributes["edittoken"].Value;
string pageFileName = directory + EscapePath(title);
string text = LoadPageFromCache(pageFileName, node.Attributes["lastrevid"].Value, title);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + title + "...");
text = wiki.LoadText(title);
CachePage(pageFileName, node.Attributes["lastrevid"].Value, text);
}
WikiPage page = WikiPage.Parse(title, text);
page.BaseTimestamp = baseTimeStamp;
page.Token = editToken;
return page;
}
public override Dictionary<string, string> Process(Wiki wiki, WikiPage page, ref int diffSize, ref int topicsArchived)
{
List<WikiPageSection> archivedSections = new List<WikiPageSection>();
foreach (WikiPageSection section in page.Sections)
{
WikiPageSection result = section.Subsections.FirstOrDefault(ss => ss.Title.Trim().ToLower() == "итог");
bool forceArchivation = LookForLines.Any(s => section.Text.ToLower().Contains(s.ToLower()));
if (!OnHold.Any(s => section.Text.ToLower().Contains(s.ToLower())) &&
((result != null && !string.IsNullOrEmpty(result.SectionText.Trim())) ||
forceArchivation ||
!CheckForResult))
{
MatchCollection ms = timeRE.Matches(FilterQuotes(section.Text));
DateTime published = DateTime.Today;
DateTime lastReply = DateTime.MinValue;
foreach (Match match in ms)
{
string value = match.Groups[1].Value;
DateTime time = DateTime.Parse(value, L10i.Culture,
DateTimeStyles.AssumeUniversal);
if (time < published)
{
published = time;
}
if (time > lastReply)
{
lastReply = time;
}
}
if (lastReply != DateTime.MinValue &&
((forceArchivation && (DateTime.Today - lastReply).TotalHours >= ForcedArchivationDelay) ||
(DateTime.Today - lastReply).TotalHours >= Delay))
{
archivedSections.Add(section);
}
}
if (IsMovedSection(section))
{
section.SectionText = section.SectionText.Trim(new char[] { ' ', '\t', '\n' }) + "\n~~~~\n";
archivedSections.Add(section);
}
}
Dictionary<string, string> archiveTexts = new Dictionary<string, string>();
if (archivedSections.Count == 0)
{
diffSize = 0;
return archiveTexts;
}
var parameters = new ParameterCollection()
{
{ "prop", "info" },
};
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, MainPage);
int ns = int.Parse(xml.SelectSingleNode("//page").Attributes["ns"].Value);
string prefix = wiki.GetNamespace(ns) + ":";
parameters = new ParameterCollection()
{
{ "list", "allpages" },
{ "apprefix", MainPage.Substring(prefix.Length) },
{ "apnamespace", ns.ToString() }
};
xml = wiki.Enumerate(parameters, true);
int maxNumber = 1;
foreach (XmlNode p in xml.SelectNodes("//p"))
{
string title = p.Attributes["title"].Value;
string format = Format.Replace("{0}", "");
if (title.StartsWith(format))
{
int number;
if (int.TryParse(title.Substring(format.Length), out number))
{
if (number > maxNumber)
{
maxNumber = number;
}
}
}
}
int index = 0;
string pageName = string.Format(Format, maxNumber);
parameters.Clear();
parameters.Add("prop", "info");
xml = wiki.Query(QueryBy.Titles, parameters, new string[] { pageName });
XmlNode node = xml.SelectSingleNode("//page");
if (node.Attributes["missing"] == null)
{
string pageFileName = _cacheDir + Cache.GenerateCachePath(pageName);
string text = Cache.LoadPageFromCache(pageFileName,
node.Attributes["lastrevid"].Value, pageName);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
Cache.CachePage(pageName, _cacheDir, node.Attributes["lastrevid"].Value, text);
}
WikiPage archivePage = WikiPage.Parse(pageName, text);
if (archivePage.Sections.Count < Topics)
{
int topics = Topics - archivePage.Sections.Count;
for (int i = 0; i < topics && index < archivedSections.Count; ++i, ++index)
{
WikiPageSection section = archivedSections[index];
section.Title = ProcessSectionTitle(section.Title);
archivePage.Sections.Add(section);
}
if (NewSectionsDown)
{
archivePage.Sections.Sort(SectionsDown);
}
else
{
archivePage.Sections.Sort(SectionsUp);
}
if (!string.IsNullOrEmpty(RemoveFromText))
{
archivePage.Text = archivePage.Text.Replace(RemoveFromText, "");
}
archiveTexts.Add(pageName, archivePage.Text);
}
}
if (index < archivedSections.Count)
{
string text = Header;
pageName = string.Format(Format, maxNumber + 1);
WikiPage archivePage = WikiPage.Parse(pageName, text);
for (; index < archivedSections.Count; ++index)
{
WikiPageSection section = archivedSections[index];
section.Title = ProcessSectionTitle(section.Title);
archivePage.Sections.Add(section);
}
archivePage.Sections.Sort(SectionsUp);
if (!string.IsNullOrEmpty(RemoveFromText))
{
archivePage.Text = archivePage.Text.Replace(RemoveFromText, "");
}
archiveTexts.Add(pageName, archivePage.Text);
}
topicsArchived = 0;
diffSize = 0;
foreach (var section in archivedSections)
{
diffSize += Encoding.UTF8.GetByteCount(section.Text);
++topicsArchived;
page.Sections.Remove(section);
}
return archiveTexts;
}
public void Analyse(Wiki wiki)
{
Directory.CreateDirectory(_cacheDir);
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", _l10i.Category);
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info");
XmlDocument doc = wiki.Enumerate(parameters, true);
XmlNodeList pages = doc.SelectNodes("//page");
List<Day> days = new List<Day>();
foreach (XmlNode page in pages)
{
string prefix = _l10i.MainPage + "/";
string pageName = page.Attributes["title"].Value;
if (pageName.Length < prefix.Length)
{
continue;
}
string date = pageName.Substring(prefix.Length);
Day day = new Day();
if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture(_l10i.Culture),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
string fileName = _cacheDir + date + ".bin";
string text = "";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
using (StreamWriter sw =
new StreamWriter(_cacheDir + "Main.txt"))
{
sw.WriteLine("{{" + _l10i.TopTemplate + "}}\n");
foreach (Day day in days)
{
sw.Write("{{" + _l10i.Template + "|" + day.Date.ToString("yyyy-M-d") + "|");
List<string> titles = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
ReplaceEmptyResults(section);
RemoveStrikeOut(section);
StrikeOutSection(section);
string result = section.Reduce("", SubsectionsList);
if (result.Length > 0)
{
result = " • <small>" + result.Substring(3) + "</small>";
}
string title;
if (_l10i.Processor != null)
{
title = _l10i.Processor(section).Trim();
}
else
{
title = section.Title.Trim();
}
titles.Add(title + result);
}
sw.Write(string.Join(" • ", titles.ConvertAll(c => c).ToArray()));
sw.Write("}}\n\n");
}
sw.WriteLine("{{" + _l10i.BottomTemplate + "}}");
}
List<string> dates = new List<string>();
DateTime today = DateTime.Today;
DateTime currentMonth = new DateTime(today.Year, today.Month, 1);
for (int i = 0; i < 2; ++i)
{
DateTime start = currentMonth.AddMonths(-i);
DateTime end = start.AddMonths(1);
while (start < end)
{
string date = start.ToString("d MMMM yyyy",
CultureInfo.CreateSpecificCulture(_l10i.Culture));
string prefix = _l10i.MainPage + "/";
string pageName = prefix + date;
dates.Add(pageName);
start = start.AddDays(1);
}
}
parameters.Clear();
parameters.Add("prop", "info");
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, dates);
days.Clear();
foreach (string pageName in dates)
{
string prefix = _l10i.MainPage + "/";
string date = pageName.Substring(prefix.Length);
string fileName = _cacheDir + date + ".bin";
bool archived = doc.SelectSingleNode("//page[@title=\"" + pageName + "\"]") == null;
XmlNode page = xml.SelectSingleNode("//page[@title=\"" + pageName + "\"]");
Day day = new Day();
day.Archived = archived;
if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture(_l10i.Culture),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
if (page.Attributes["missing"] != null)
{
day.Exists = false;
days.Add(day);
continue;
}
string text = "";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
day.Exists = true;
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
for (int i = 0; i < 2; ++i)
{
DateTime start = currentMonth.AddMonths(-i);
DateTime end = start.AddMonths(1);
string archiveDate = start.ToString("yyyy-MM");
IEnumerable<Day> daysInMonth = days.Where(d => d.Date >= start &&
d.Date < end);
using (StreamWriter sw =
new StreamWriter(_cacheDir + "Archive-" + archiveDate + ".txt"))
{
sw.WriteLine(_l10i.ArchiveHeader);
StringBuilder sb = new StringBuilder();
foreach (Day day in daysInMonth)
{
sb.Append("{{" + _l10i.Template + "|" + day.Date.ToString("yyyy-M-d") + "|");
if (!day.Exists)
{
sb.Append("''" + _l10i.EmptyArchive + "''}}\n\n");
continue;
}
List<string> titles = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
string result = section.Reduce("", SubsectionsList);
if (result.Length > 0)
{
result = " • <small>" + result.Substring(3) + "</small>";
}
string title;
if (_l10i.Processor != null)
{
title = _l10i.Processor(section).Trim();
}
else
{
title = section.Title.Trim();
}
titles.Add(title + result);
}
sb.Append(string.Join(" • ", titles.ConvertAll(c => c).ToArray()));
sb.Append("}}\n\n");
}
sb.Replace("<s>", "");
sb.Replace("</s>", "");
sb.Replace("<strike>", "");
sb.Replace("</strike>", "");
sw.Write(sb.ToString());
sw.WriteLine(_l10i.ArchiveFooter);
}
}
}
public void UpdatePages(Wiki wiki)
{
Console.Out.WriteLine("Updating articles for deletion...");
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
Regex timeRE = new Regex(@"(\d{2}:\d{2}\, \d\d? [а-я]+ \d{4}) \(UTC\)");
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", _l10i.Category);
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info|revisions");
parameters.Add("intoken", "edit");
XmlDocument doc = wiki.Enumerate(parameters, true);
string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
XmlNodeList pages = doc.SelectNodes("//page");
foreach (XmlNode page in pages)
{
string starttimestamp = queryTimestamp;
int results = 0;
string prefix = _l10i.MainPage + "/";
string pageName = page.Attributes["title"].Value;
string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value;
string editToken = page.Attributes["edittoken"].Value;
if (pageName.Length < prefix.Length)
{
continue;
}
string date = pageName.Substring(prefix.Length);
Day day = new Day();
if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture(_l10i.Culture),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
string text = "";
string fileName = _cacheDir + date + ".bin";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
try
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
starttimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
}
catch (WikiPageNotFound)
{
continue;
}
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
List<string> titlesWithResults = new List<string>();
Dictionary<string, List<WikiPageSection>> titles = new Dictionary<string, List<WikiPageSection>>();
day.Page = WikiPage.Parse(pageName, text);
foreach (WikiPageSection section in day.Page.Sections)
{
ReplaceEmptyResults(section);
RemoveStrikeOut(section);
StrikeOutSection(section);
if (section.Subsections.Count(s => _l10i.Processor != null
? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim())
: _l10i.Results.Any(r => r == s.Title.Trim())) == 0 &&
section.Subsections.Count(s => s.Title.Trim() == _l10i.ChallengedResult) == 0)
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(section);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(section);
}
}
}
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success && section.Title.Contains("<s>"))
{
titlesWithResults.Add(m.Groups[1].Value.Trim());
}
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
m = wikiLinkRE.Match(subsection.Title);
if (m.Success && subsection.Title.Contains("<s>"))
{
titlesWithResults.Add(m.Groups[1].Value.Trim());
}
if (m.Success &&
!subsection.Title.Contains("<s>") &&
subsection.Subsections.Count(s => s.Title.Trim() == _l10i.ChallengedResult) == 0 &&
subsection.Subsections.Count(s => _l10i.Processor != null
? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim())
: _l10i.Results.Any(r => r == s.Title.Trim())) == 0)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(subsection);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(subsection);
}
}
}
}
}
parameters.Clear();
parameters.Add("prop", "info");
Dictionary<string, string> normalizedTitles = new Dictionary<string, string>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles.Keys);
foreach (XmlNode node in xml.SelectNodes("//n"))
{
normalizedTitles.Add(node.Attributes["to"].Value,
node.Attributes["from"].Value);
}
List<string> notificationList = new List<string>();
XmlNodeList missingTitles = xml.SelectNodes("//page");
foreach (XmlNode node in missingTitles)
{
string title = node.Attributes["title"].Value;
IEnumerable<WikiPageSection> sections;
if (titles.ContainsKey(title))
{
sections = titles[title];
}
else
{
sections = titles[normalizedTitles[title]];
}
if (node.Attributes["missing"] != null)
{
DateTime start = day.Date;
//foreach (WikiPageSection section in sections)
//{
// Match m = timeRE.Match(section.Text);
// if (m.Success)
// {
// start = DateTime.Parse(m.Groups[1].Value,
// CultureInfo.CreateSpecificCulture(_l10i.Culture),
// DateTimeStyles.AssumeUniversal);
// }
//}
parameters.Clear();
parameters.Add("list", "logevents");
parameters.Add("letype", "delete");
parameters.Add("lemlimit", "max");
parameters.Add("lestart", start.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"));
parameters.Add("ledir", "newer");
parameters.Add("letitle", title);
parameters.Add("leprop", "comment|type|user|timestamp");
XmlDocument log = wiki.Enumerate(parameters, true);
XmlNodeList items = log.SelectNodes("//item");
List<DeleteLogEvent> events = new List<DeleteLogEvent>();
foreach (XmlNode item in items)
{
DeleteLogEvent ev = new DeleteLogEvent();
ev.Comment = item.Attributes["comment"].Value;
ev.Deleted = item.Attributes["action"].Value == "delete";
ev.User = item.Attributes["user"].Value;
ev.Timestamp = DateTime.Parse(item.Attributes["timestamp"].Value,
null,
DateTimeStyles.AssumeUniversal);
events.Add(ev);
}
events.Sort(CompareDeleteLogEvents);
if (events.Count > 0 &&
events[0].Deleted &&
(DateTime.Now - events[0].Timestamp).TotalHours > 2)
{
string comment = FilterWikiMarkup(events[0].Comment);
string message = string.Format(_l10i.AutoResultMessage,
events[0].User,
events[0].Timestamp.ToUniversalTime().ToString(_l10i.DateFormat, CultureInfo.CreateSpecificCulture(_l10i.Culture)),
comment);
var pageSections = titles.ContainsKey(title)
? titles[title]
: titles[normalizedTitles[title]];
foreach (WikiPageSection section in pageSections)
{
WikiPageSection verdict = new WikiPageSection(" " + _l10i.AutoResultSection + " ",
section.Level + 1,
message);
section.AddSubsection(verdict);
StrikeOutSection(section);
++results;
}
}
}
}
if (_l10i.Culture != "ru-RU")
{
parameters.Clear();
parameters.Add("prop", "info");
xml = wiki.Query(QueryBy.Titles, parameters, titlesWithResults);
foreach (XmlNode node in xml.SelectNodes("//page"))
{
if (node.Attributes["missing"] == null &&
node.Attributes["redirect"] == null &&
node.Attributes["ns"].Value == "0")
{
notificationList.Add(node.Attributes["title"].Value);
}
}
if (notificationList.Count > 0)
{
parameters.Clear();
parameters.Add("list", "backlinks");
parameters.Add("bltitle", pageName);
parameters.Add("blfilterredir", "nonredirects");
parameters.Add("blnamespace", "1");
parameters.Add("bllimit", "max");
XmlDocument backlinks = wiki.Enumerate(parameters, true);
foreach (string title in notificationList)
{
string talkPage = wiki.GetNamespace(1) + ":" + title;
if (backlinks.SelectSingleNode("//bl[@title=" + GenerateConcatForXPath(talkPage) + "]") == null)
{
PutNotification(wiki, title, date);
}
}
}
}
string newText = day.Page.Text;
if (newText.Trim() == text.Trim())
{
continue;
}
try
{
Console.Out.WriteLine("Updating " + pageName + "...");
string revid = wiki.Save(pageName,
"",
newText,
_l10i.StrikeOutComment + (results > 0 ? _l10i.AutoResultComment : ""),
MinorFlags.Minor,
CreateFlags.NoCreate,
WatchFlags.None,
SaveFlags.Replace,
true,
basetimestamp,
"",
editToken);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(revid);
sw.Write(newText);
}
}
catch (WikiException)
{
}
}
}
public void UpdateArchivePages(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", _l10i.Category);
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info");
XmlDocument doc = wiki.Enumerate(parameters, true);
XmlNodeList pages = doc.SelectNodes("//page");
DateTime minDate = DateTime.Now;
foreach (XmlNode page in pages)
{
string prefix = _l10i.MainPage + "/";
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring(prefix.Length);
DateTime day;
if (DateTime.TryParse(date,
_l10i.Culture,
DateTimeStyles.AssumeUniversal, out day))
{
if (day < minDate)
{
minDate = day;
}
}
else
{
continue;
}
}
List<string> titles = new List<string>();
minDate = new DateTime(minDate.Year, minDate.Month, 1);
DateTime currentMonth = new DateTime(DateTime.Today.Year,
DateTime.Today.Month, 1);
DateTime start = minDate;
while (start <= currentMonth)
{
string date = start.ToString("yyyy-MM");
string pageName = _l10i.ArchivePage + date;
titles.Add(pageName);
start = start.AddMonths(1);
}
parameters.Clear();
parameters.Add("prop", "info");
XmlDocument archivesDoc = wiki.Query(QueryBy.Titles, parameters, titles);
pages = archivesDoc.SelectNodes("//page");
foreach (XmlNode archivePage in pages)
{
string archiveName = archivePage.Attributes["title"].Value;
string date = archiveName.Substring(_l10i.ArchivePage.Length);
DateTime archiveDate;
if (!DateTime.TryParse(date,
_l10i.Culture,
DateTimeStyles.AssumeUniversal, out archiveDate))
{
continue;
}
string fileName = _cacheDir + "Archive-" + date + ".txt";
start = archiveDate;
DateTime end = start.AddMonths(1);
titles.Clear();
while (start < end)
{
string pageDate = start.ToString("d MMMM yyyy",
_l10i.Culture);
string prefix = _l10i.MainPage + "/";
string pageName = prefix + pageDate;
titles.Add(pageName);
start = start.AddDays(1);
}
parameters.Clear();
parameters.Add("prop", "info");
List<Day> days = new List<Day>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles);
XmlNodeList archives = xml.SelectNodes("//page");
foreach (XmlNode page in archives)
{
string prefix = _l10i.MainPage + "/";
string pageName = page.Attributes["title"].Value;
string dateString = pageName.Substring(prefix.Length);
string pageFileName = _cacheDir + dateString + ".bin";
Day day = new Day();
day.Archived = doc.SelectSingleNode("//page[@title=\"" + pageName + "\"]") == null;
if (!DateTime.TryParse(dateString,
_l10i.Culture,
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
if (page.Attributes["missing"] != null)
{
day.Exists = false;
days.Add(day);
continue;
}
string text = LoadPageFromCache(pageFileName,
page.Attributes["lastrevid"].Value, pageName);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
CachePage(pageFileName, page.Attributes["lastrevid"].Value, text);
}
day.Exists = true;
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
StringBuilder textBuilder = new StringBuilder();
textBuilder.AppendLine(_l10i.ArchiveHeader);
StringBuilder sb = new StringBuilder();
foreach (Day day in days)
{
sb.Append("{{" + _l10i.Template + "|" + day.Date.ToString("yyyy-M-d") + "|\n");
if (!day.Exists)
{
sb.Append("''" + _l10i.EmptyArchive + "''}}\n\n");
continue;
}
List<string> sectionTitles = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
string result = section.Reduce("", SubsectionsList);
if (result.Length > 0)
{
result = " • <small>" + result.Substring(3) + "</small>";
}
string title;
if (_l10i.Processor != null)
{
title = _l10i.Processor(section).Trim();
}
else
{
title = section.Title.Trim();
}
sectionTitles.Add(title + result);
}
sb.Append(string.Join(" • ", sectionTitles.ConvertAll(c => c).ToArray()));
sb.Append("}}\n\n");
}
sb.Replace("<s>", "");
sb.Replace("</s>", "");
sb.Replace("<strike>", "");
sb.Replace("</strike>", "");
textBuilder.Append(sb.ToString());
textBuilder.AppendLine(_l10i.ArchiveFooter);
if (File.Exists(fileName))
{
using (TextReader sr = new StreamReader(fileName))
{
string text = sr.ReadToEnd();
if (text == textBuilder.ToString())
{
continue;
}
}
}
Console.Out.WriteLine("Updating " + archiveName + "...");
wiki.Save(archiveName,
textBuilder.ToString(),
_l10i.MainPageUpdateComment);
using (StreamWriter sw =
new StreamWriter(fileName))
{
sw.Write(textBuilder.ToString());
}
}
}
public void UpdatePages(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения объединения страниц");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info");
XmlDocument doc = wiki.Enumerate(parameters, true);
XmlNodeList pages = doc.SelectNodes("//page");
foreach (XmlNode page in pages)
{
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring("Википедия:К объединению/".Length);
Day day = new Day();
try
{
day.Date = DateTime.Parse(date,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
catch (FormatException)
{
continue;
}
string fileName = _cacheDir + date + ".bin";
string text = LoadPageFromCache(fileName,
page.Attributes["lastrevid"].Value, pageName);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
CachePage(fileName, page.Attributes["lastrevid"].Value, text);
}
day.Page = WikiPage.Parse(pageName, text);
foreach (WikiPageSection section in day.Page.Sections)
{
RemoveStrikeOut(section);
StrikeOutSection(section);
}
string newText = day.Page.Text;
if (newText.Trim() == text.Trim())
{
continue;
}
try
{
string revid = wiki.Save(pageName,
newText,
"зачёркивание заголовков");
CachePage(fileName, revid, newText);
}
catch (WikiException)
{
}
}
}
public void UpdatePages(Wiki wiki)
{
Regex closedRE = new Regex(@"(\{{2}ВПКПМ-Навигация\}{2}\s*\{{2}(Закрыто|Closed|закрыто|closed)\}{2})|(\{{2}(Закрыто|Closed|закрыто|closed)\}{2}\s*\{{2}ВПКПМ-Навигация\}{2})");
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
Regex timeRE = new Regex(@"(\d{2}:\d{2}\, \d\d? [а-я]+ \d{4}) \(UTC\)");
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения переименования страниц");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info|revisions");
parameters.Add("intoken", "edit");
XmlDocument doc = wiki.Enumerate(parameters, true);
string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
XmlNodeList pages = doc.SelectNodes("//page");
foreach (XmlNode page in pages)
{
int results = 0;
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring("Википедия:К переименованию/".Length);
string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value;
string editToken = page.Attributes["edittoken"].Value;
Day day = new Day();
if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
string fileName = _cacheDir + date + ".bin";
string text = "";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
day.Page = WikiPage.Parse(pageName, text);
List<string> titlesWithResults = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
RemoveStrikeOut(section);
StrikeOutSection(section);
Match m = wikiLinkRE.Match(section.Title);
if (m.Success && !m.Groups[1].Value.StartsWith(":Категория:"))
{
string link = m.Groups[1].Value;
string movedTo;
string movedBy;
DateTime movedAt;
DateTime start = day.Date;
m = timeRE.Match(section.SectionText);
if (m.Success)
{
start = DateTime.Parse(m.Groups[1].Value,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
bool moved = MovedTo(wiki,
link,
start,
out movedTo,
out movedBy,
out movedAt);
WikiPageSection autoVerdictSection = section.Subsections.FirstOrDefault(s => s.Title.Trim() == "Автоматический итог");
if (autoVerdictSection != null && !moved)
{
autoVerdictSection.Title = " Оспоренный итог ";
RemoveStrikeOut(section);
StrikeOutSection(section);
}
if (section.Subsections.Count(s => s.Title.Trim() == "Оспоренный итог") == 0 &&
section.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") == 0 &&
section.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") == 0 &&
moved && !string.IsNullOrEmpty(movedTo))
{
string message = string.Format("Страница была переименована {2} в «[[{0}]]» участником [[User:{1}|]]. Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
movedTo,
movedBy,
movedAt.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)"));
WikiPageSection verdict = new WikiPageSection(" Автоматический итог ",
section.Level + 1,
message);
section.AddSubsection(verdict);
StrikeOutSection(section);
++results;
}
}
m = wikiLinkRE.Match(section.Title);
if (m.Success && section.Title.Contains("<s>"))
{
titlesWithResults.Add(m.Groups[1].Value);
}
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
m = wikiLinkRE.Match(subsection.Title);
if (m.Success && subsection.Title.Contains("<s>"))
{
titlesWithResults.Add(m.Groups[1].Value.Trim());
}
if (m.Success &&
!subsection.Title.Contains("<s>") &&
!m.Groups[1].Value.StartsWith(":Категория:"))
{
string link = m.Groups[1].Value;
string movedTo;
string movedBy;
DateTime movedAt;
DateTime start = day.Date;
m = timeRE.Match(subsection.SectionText);
if (m.Success)
{
start = DateTime.Parse(m.Groups[1].Value,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
bool moved = MovedTo(wiki,
link,
start,
out movedTo,
out movedBy,
out movedAt);
WikiPageSection verdictSection = subsection.Subsections.FirstOrDefault(s => s.Title.Trim() == "Автоматический итог");
if (verdictSection != null && !moved)
{
verdictSection.Title = " Оспоренный итог ";
RemoveStrikeOut(subsection);
StrikeOutSection(subsection);
}
if (subsection.Subsections.Count(s => s.Title.Trim() == "Оспоренный итог") == 0 &&
subsection.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") == 0 &&
subsection.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") == 0 &&
moved && !string.IsNullOrEmpty(movedTo))
{
string message = string.Format("Страница была переименована {2} в «[[{0}]]» участником [[User:{1}|]]. Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
movedTo,
movedBy,
movedAt.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)"));
WikiPageSection verdict = new WikiPageSection(" Автоматический итог ",
subsection.Level + 1,
message);
subsection.AddSubsection(verdict);
StrikeOutSection(subsection);
++results;
}
}
}
}
Match matchClosed = closedRE.Match(text);
if (matchClosed.Success)
{
List<TalkResult> talkResults = new List<TalkResult>();
foreach (string name in titlesWithResults)
{
if (wiki.PageNamespace(name) > 0)
{
continue;
}
string movedTo;
string movedBy;
DateTime movedAt;
bool moved = MovedTo(wiki,
name,
day.Date,
out movedTo,
out movedBy,
out movedAt);
talkResults.Add(new TalkResult(name, movedTo, moved));
}
parameters.Clear();
parameters.Add("prop", "info");
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, talkResults.ConvertAll(r => r.Moved ? r.MovedTo : r.Title));
List<string> notificationList = new List<string>();
foreach (XmlNode node in xml.SelectNodes("//page"))
{
if (node.Attributes["missing"] == null && node.Attributes["ns"].Value == "0")
{
notificationList.Add(node.Attributes["title"].Value);
}
}
if (notificationList.Count > 0)
{
parameters.Clear();
parameters.Add("list", "backlinks");
parameters.Add("bltitle", pageName);
parameters.Add("blfilterredir", "nonredirects");
parameters.Add("blnamespace", "1");
parameters.Add("bllimit", "max");
XmlDocument backlinks = wiki.Enumerate(parameters, true);
foreach (string title in notificationList)
{
if (backlinks.SelectSingleNode("//bl[@title=\"Обсуждение:" + title + "\"]") == null)
{
TalkResult tr = talkResults.Find(r => r.Moved ? r.MovedTo == title : r.Title == title);
PutNotification(wiki, tr, day.Date);
}
}
}
}
string newText = day.Page.Text;
if (newText.Trim() == text.Trim())
{
continue;
}
try
{
Console.Out.WriteLine("Updating " + pageName + "...");
string revid = wiki.Save(pageName,
"",
newText,
"зачёркивание заголовков" + (results > 0 ? ", сообщение об итогах" : ""),
MinorFlags.Minor,
CreateFlags.NoCreate,
WatchFlags.None,
SaveFlags.Replace,
true,
basetimestamp,
"",
editToken);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(revid);
sw.Write(newText);
}
}
catch (WikiException)
{
}
}
}
public void UpdateArchivePages(Wiki wiki, int year, int monthNumber)
{
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
DateTime month = new DateTime(year, monthNumber, 1);
DateTime end = month.AddMonths(1);
using (StreamWriter archiveSW =
new StreamWriter(_cacheDir + "Archive-" +
year.ToString() + "-" + monthNumber.ToString() + ".txt"))
while (month < end && month < DateTime.Now)
{
DateTime start = month;
DateTime nextMonth = start.AddMonths(1);
List<string> titles = new List<string>();
while (start < nextMonth)
{
string pageDate = start.ToString("d MMMM yyyy",
CultureInfo.CreateSpecificCulture("ru-RU"));
string prefix = "Википедия:К переименованию/";
string pageName = prefix + pageDate;
titles.Add(pageName);
start = start.AddDays(1);
}
ParameterCollection parameters = new ParameterCollection();
parameters.Add("prop", "info");
List<Day> days = new List<Day>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles);
XmlNodeList archives = xml.SelectNodes("//page");
foreach (XmlNode page in archives)
{
string pageName = page.Attributes["title"].Value;
string dateString = pageName.Substring("Википедия:К переименованию/".Length);
string pageFileName = _cacheDir + dateString + ".bin";
Day day = new Day();
try
{
day.Date = DateTime.Parse(dateString,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
catch (FormatException)
{
continue;
}
if (page.Attributes["missing"] != null)
{
day.Exists = false;
days.Add(day);
continue;
}
string text = "";
if (File.Exists(pageFileName))
{
using (FileStream fs = new FileStream(pageFileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
using (FileStream fs = new FileStream(pageFileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
day.Exists = true;
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
StringBuilder textBuilder = new StringBuilder();
textBuilder.AppendLine("{{Навигация по архиву КПМ}}\n{{Переименование статей/Начало}}");
StringBuilder sb = new StringBuilder();
foreach (Day day in days)
{
sb.Append("{{Переименование статей/День|" + day.Date.ToString("yyyy-M-d") + "|\n");
if (!day.Exists)
{
sb.Append("''нет обсуждений''}}\n\n");
continue;
}
titles.Clear();
Console.Out.WriteLine("Analyzing " + day.Date.ToString("d MMMM yyyy") + "...");
foreach (WikiPageSection section in day.Page.Sections)
{
string filler = "";
string result = "";
bool hasVerdict = section.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") > 0;
bool hasAutoVerdict = section.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") > 0;
if (hasVerdict || hasAutoVerdict || section.Title.Contains("<s>"))
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success && !m.Groups[1].Value.StartsWith(":Категория:"))
{
string link = m.Groups[1].Value;
string movedTo;
bool moved = MovedTo(wiki, link, day.Date, out movedTo);
if (moved && string.IsNullOrEmpty(movedTo))
{
result = " ''(переименовано)''";
}
else if (moved)
{
result = string.Format(" ''({1}переименовано в «[[{0}]]»)''",
movedTo.StartsWith("Файл:") ? ":" + movedTo : movedTo, hasAutoVerdict && !hasVerdict ? "де-факто " : "");
}
else
{
result = " ''(не переименовано)''";
}
}
}
for (int i = 0; i < section.Level - 1; ++i)
{
filler += "*";
}
titles.Add(filler + " " + section.Title.Trim() + result);
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
result = "";
hasVerdict = subsection.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") > 0;
hasAutoVerdict = subsection.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") > 0;
if (hasVerdict || hasAutoVerdict || subsection.Title.Contains("<s>"))
{
Match m = wikiLinkRE.Match(subsection.Title);
if (m.Success && !m.Groups[1].Value.StartsWith(":Категория:"))
{
string link = m.Groups[1].Value;
string movedTo;
bool moved = MovedTo(wiki, link, day.Date, out movedTo);
if (moved && string.IsNullOrEmpty(movedTo))
{
result = " ''(переименовано)''";
}
else if (moved)
{
result = string.Format(" ''({1}переименовано в «[[{0}]]»)''",
movedTo.StartsWith("Файл:") ? ":" + movedTo : movedTo, hasAutoVerdict && !hasVerdict ? "де-факто " : "");
}
else
{
result = " ''(не переименовано)''";
}
}
}
filler = "";
for (int i = 0; i < subsection.Level - 1; ++i)
{
filler += "*";
}
titles.Add(filler + " " + subsection.Title.Trim() + result);
}
}
if (titles.Count(s => s.Contains("=")) > 0)
{
titles[0] = "2=<li>" + titles[0].Substring(2) + "</li>";
}
sb.Append(string.Join("\n", titles.ConvertAll(c => c).ToArray()));
sb.Append("}}\n\n");
}
sb.Replace("<s>", "");
sb.Replace("</s>", "");
sb.Replace("<strike>", "");
sb.Replace("</strike>", "");
textBuilder.Append(sb.ToString());
textBuilder.AppendLine("{{Переименование статей/Конец}}");
archiveSW.WriteLine(textBuilder.ToString());
month = month.AddMonths(1);
}
string archiveName = string.Format("Википедия:Архив запросов на переименование/{0}-{1:00}",
year, monthNumber);
Console.Out.WriteLine("Updating " + archiveName + "...");
using (TextReader sr =
new StreamReader(_cacheDir + "Archive-" +
year.ToString() + "-" + monthNumber.ToString() + ".txt"))
{
string text = sr.ReadToEnd();
wiki.Save(archiveName, text, "обновление");
}
}
public override Dictionary<string, string> Process(Wiki wiki, WikiPage page, ref int diffSize, ref int topics)
{
Dictionary<string, string> results = new Dictionary<string, string>();
Dictionary<DateTime, List<WikiPageSection>> archives = new Dictionary<DateTime, List<WikiPageSection>>();
List<WikiPageSection> archivedSections = new List<WikiPageSection>();
foreach (WikiPageSection section in page.Sections)
{
WikiPageSection result = section.Subsections.FirstOrDefault(ss => ss.Title.Trim().ToLower() == "итог");
bool forceArchivation = LookForLines.Any(s => section.Text.ToLower().Contains(s.ToLower()));
if (!OnHold.Any(s => section.Text.ToLower().Contains(s.ToLower())) &&
((result != null && !string.IsNullOrEmpty(result.SectionText.Trim())) ||
forceArchivation ||
!CheckForResult))
{
MatchCollection ms = timeRE.Matches(FilterQuotes(section.Text));
DateTime published = NormalizeDate(DateTime.Today);
DateTime lastReply = DateTime.MinValue;
foreach (Match match in ms)
{
string value = match.Groups[1].Value;
DateTime time;
if (!DateTime.TryParse(value, L10i.Culture,
DateTimeStyles.AssumeUniversal, out time))
{
continue;
}
if (time < published)
{
published = NormalizeDate(time);
}
if (time > lastReply)
{
lastReply = time;
}
}
if (lastReply != DateTime.MinValue &&
((forceArchivation && (DateTime.Today - lastReply).TotalHours >= ForcedArchivationDelay) ||
(DateTime.Today - lastReply).TotalHours >= Delay))
{
if (archives.ContainsKey(published))
{
archives[published].Add(section);
}
else
{
List<WikiPageSection> sections = new List<WikiPageSection>();
sections.Add(section);
archives.Add(published, sections);
}
archivedSections.Add(section);
}
}
if (IsMovedSection(section))
{
section.SectionText = section.SectionText.Trim(new char[] { ' ', '\t', '\n' }) + "\n~~~~\n";
}
}
if (archivedSections.Count == 0)
{
return results;
}
foreach (DateTime period in archives.Keys)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("prop", "info");
string pageName = DateToPageName(period);
string pageFileName = _cacheDir + Cache.GenerateCachePath(period.ToString(Format));
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, new string[] { pageName });
XmlNode node = xml.SelectSingleNode("//page");
string text;
if (node.Attributes["missing"] == null)
{
text = Cache.LoadPageFromCache(pageFileName,
node.Attributes["lastrevid"].Value, pageName);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
Cache.CachePage(period.ToString(Format), _cacheDir, node.Attributes["lastrevid"].Value, text);
}
}
else
{
text = Header;
}
WikiPage archivePage = WikiPage.Parse(pageName, text);
foreach (WikiPageSection section in archives[period])
{
section.Title = ProcessSectionTitle(section.Title);
archivePage.Sections.Add(section);
}
if (NewSectionsDown)
{
archivePage.Sections.Sort(SectionsDown);
}
else
{
archivePage.Sections.Sort(SectionsUp);
}
if (!string.IsNullOrEmpty(RemoveFromText))
{
archivePage.Text = archivePage.Text.Replace(RemoveFromText, "");
}
results.Add(pageName, archivePage.Text);
}
topics = 0;
diffSize = 0;
foreach (var section in archivedSections)
{
diffSize += Encoding.UTF8.GetByteCount(section.Text);
++topics;
page.Sections.Remove(section);
}
return results;
}
internal void UpdatePages(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения восстановления страниц");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info|revisions");
parameters.Add("intoken", "edit");
XmlDocument doc = wiki.Enumerate(parameters, true);
string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
XmlNodeList pages = doc.SelectNodes("//page");
foreach (XmlNode page in pages)
{
string starttimestamp = queryTimestamp;
int results = 0;
string prefix = "Википедия:К восстановлению/";
string pageName = page.Attributes["title"].Value;
string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value;
string editToken = page.Attributes["edittoken"].Value;
string date = pageName.Substring(prefix.Length);
Day day = new Day();
if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
string text = "";
string fileName = _cacheDir + date + ".bin";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
try
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
starttimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
}
catch (WikiPageNotFound)
{
continue;
}
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
Dictionary<string, List<WikiPageSection>> titles = new Dictionary<string, List<WikiPageSection>>();
day.Page = WikiPage.Parse(pageName, text);
foreach (WikiPageSection section in day.Page.Sections)
{
RemoveStrikeOut(section);
StrikeOutSection(section);
if (section.Subsections.Count(s => _results.Any(r => r.ToLower() == s.Title.Trim().ToLower())) == 0)
{
Match m = _wikiLinkRE.Match(section.Title);
if (m.Success)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(section);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(section);
}
}
}
{
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
Match m = _wikiLinkRE.Match(subsection.Title);
if (m.Success &&
!subsection.Title.Contains("<s>") &&
subsection.Subsections.Count(s => _results.Any(r => r.ToLower() == s.Title.Trim().ToLower())) == 0)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(subsection);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(subsection);
}
}
}
}
}
parameters.Clear();
parameters.Add("prop", "info");
Dictionary<string, string> normalizedTitles = new Dictionary<string, string>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles.Keys);
foreach (XmlNode node in xml.SelectNodes("//n"))
{
normalizedTitles.Add(node.Attributes["to"].Value,
node.Attributes["from"].Value);
}
List<string> notificationList = new List<string>();
XmlNodeList missingTitles = xml.SelectNodes("//page");
foreach (XmlNode node in missingTitles)
{
string title = node.Attributes["title"].Value;
IEnumerable<WikiPageSection> sections;
if (titles.ContainsKey(title))
{
sections = titles[title];
}
else
{
sections = titles[normalizedTitles[title]];
}
if (node.Attributes["missing"] == null)
{
DateTime start = day.Date;
parameters.Clear();
parameters.Add("list", "logevents");
parameters.Add("letype", "delete");
parameters.Add("lemlimit", "max");
parameters.Add("lestart", start.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"));
parameters.Add("ledir", "newer");
parameters.Add("letitle", title);
parameters.Add("leprop", "comment|type|user|timestamp");
XmlDocument log = wiki.Enumerate(parameters, true);
XmlNodeList items = log.SelectNodes("//item");
List<DeleteLogEvent> events = new List<DeleteLogEvent>();
foreach (XmlNode item in items)
{
DeleteLogEvent ev = new DeleteLogEvent();
ev.Comment = item.Attributes["comment"].Value;
ev.Restored = item.Attributes["action"].Value == "restore";
ev.Deleted = item.Attributes["action"].Value == "delete";
ev.User = item.Attributes["user"].Value;
ev.Timestamp = DateTime.Parse(item.Attributes["timestamp"].Value,
null,
DateTimeStyles.AssumeUniversal);
events.Add(ev);
}
events.Sort(CompareDeleteLogEvents);
if (events.Count > 0 &&
events[0].Restored &&
(DateTime.Now - events[0].Timestamp).TotalHours > 2)
{
Regex commentRE = new Regex(@" восстановлено: (.+)");
Match m = commentRE.Match(events[0].Comment);
string comment;
if (m.Success)
{
comment = m.Groups[1].Value;
}
else
{
comment = "<nowiki>" + events[0].Comment + "</nowiki>";
}
string message = string.Format("Страница была восстановлена {1} администратором [[User:{0}|]]. Была указана следующая причина: «{2}». Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
events[0].User,
events[0].Timestamp.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)"),
comment);
if (!titles.ContainsKey(title))
{
continue;
}
foreach (WikiPageSection section in titles[title])
{
WikiPageSection verdict = new WikiPageSection(" Автоматический итог ",
section.Level + 1,
message);
section.AddSubsection(verdict);
StrikeOutSection(section);
++results;
}
}
else if (events.Count > 0 &&
events[0].Deleted)
{
parameters.Clear();
parameters.Add("prop", "revisions");
parameters.Add("rvprop", "timestamp|user");
parameters.Add("rvdir", "newer");
parameters.Add("rvlimit", "1");
parameters.Add("redirects");
log = wiki.Query(QueryBy.Titles, parameters, new string[] { title });
XmlNode revision = log.SelectSingleNode("//rev");
if (revision != null)
{
string user = revision.Attributes["user"].Value;
string timestamp = revision.Attributes["timestamp"].Value;
DateTime time = DateTime.Parse(timestamp,
null,
DateTimeStyles.AssumeUniversal);
string message = string.Format("Страница была создана заново {1} участником [[User:{0}|]]. Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
user,
time.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)"));
if (!titles.ContainsKey(title))
{
continue;
}
foreach (WikiPageSection section in titles[title])
{
WikiPageSection verdict = new WikiPageSection(" Автоматический итог ",
section.Level + 1,
message);
section.AddSubsection(verdict);
StrikeOutSection(section);
++results;
}
}
}
}
}
string newText = day.Page.Text;
if (newText.Trim() == text.Trim())
{
continue;
}
try
{
Console.Out.WriteLine("Updating " + pageName + "...");
string revid = wiki.Save(pageName,
"",
newText,
"зачёркивание заголовков" + (results > 0 ? " и подведение итогов" : ""),
MinorFlags.Minor,
CreateFlags.NoCreate,
WatchFlags.None,
SaveFlags.Replace,
true,
basetimestamp,
"",
editToken);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(revid);
sw.Write(newText);
}
}
catch (WikiException)
{
}
}
}
public void Analyze(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения восстановления страниц");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info|revisions");
parameters.Add("rvprop", "timestamp");
XmlDocument doc = wiki.Enumerate(parameters, true);
XmlNodeList pages = doc.SelectNodes("//page");
List<Day> days = new List<Day>();
DateTime start = DateTime.Today;
Regex closedRE = new Regex(@"({{ВПВУС-Навигация}}\s*{{(Закрыто|Closed|закрыто|closed)}})|({{(Закрыто|Closed|закрыто|closed)}}\s*{{ВПВУС-Навигация}})");
foreach (XmlNode page in pages)
{
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring("Википедия:К восстановлению/".Length);
DateTime cutOffDate = new DateTime(2008, 11, 13);
Day day = new Day();
if (!DateTime.TryParse(date,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
string fileName = _cacheDir + date + ".bin";
string text = "";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
DateTime lastEdit = DateTime.Parse(page.FirstChild.FirstChild.Attributes["timestamp"].Value, null, DateTimeStyles.AssumeUniversal);
Match m = closedRE.Match(text);
if ((DateTime.Now - lastEdit).TotalDays > 2 && (m.Success || day.Date < cutOffDate))
{
Console.Out.WriteLine("Closing " + pageName + "...");
text = text.Replace("{{ВПВУС-Навигация}}", "{{ВПВУС-Навигация|nocat=1}}");
wiki.Save(pageName,
text,
"обсуждение закрыто");
continue;
}
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
List<string> sectionTitles = new List<string>();
foreach (Day day in days)
{
foreach (WikiPageSection section in day.Page.Sections)
{
RemoveStrikeOut(section);
StrikeOutSection(section);
if (section.Title.Contains("<s>"))
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success)
{
sectionTitles.Add(m.Groups[1].Value);
}
}
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
if (subsection.Title.Contains("<s>"))
{
Match m = wikiLinkRE.Match(subsection.Title);
if (m.Success)
{
sectionTitles.Add(m.Groups[1].Value);
}
}
}
}
}
parameters.Clear();
parameters.Add("prop", "info");
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, sectionTitles);
using (StreamWriter sw =
new StreamWriter(_cacheDir + "MainPage.txt"))
{
sw.WriteLine("== Текущие обсуждения ==\n");
sw.WriteLine("{{Восстановление статей/Статьи, вынесенные на обсуждение восстановления}}\n");
foreach (Day day in days)
{
sw.Write("{{Восстановление статей/День|" + day.Date.ToString("yyyy-M-d") + "|\n");
List<string> titles = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
string result = "";
string filler = "";
RemoveStrikeOut(section);
StrikeOutSection(section);
if (section.Title.Contains("<s>"))
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success)
{
string link = m.Groups[1].Value;
XmlNode node = xml.SelectSingleNode("//page[@title='" + link.Replace("'", @"'") + "']");
if (node != null)
{
if (node.Attributes["missing"] == null)
{
WikiPageSection autoresult = section.Subsections.FirstOrDefault(s => s.Title.Trim() == "Автоматический итог");
if (autoresult != null &&
autoresult.SectionText.Contains("Страница была создана заново"))
{
result = " ''(создана заново)''";
}
else
{
result = " ''(восстановлена)''";
}
}
else
{
result = " ''(не восстановлена)''";
}
}
}
}
for (int i = 0; i < section.Level - 1; ++i)
{
filler += "*";
}
titles.Add(filler + " " + section.Title.Trim() + result);
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
result = "";
if (subsection.Title.Contains("<s>"))
{
Match m = wikiLinkRE.Match(subsection.Title);
if (m.Success)
{
string link = m.Groups[1].Value;
XmlNode node = xml.SelectSingleNode("//page[@title='" + link + "']");
if (node != null)
{
if (node.Attributes["missing"] == null)
{
WikiPageSection autoresult = subsection.Subsections.FirstOrDefault(s => s.Title.Trim() == "Автоматический итог");
if (autoresult != null &&
autoresult.SectionText.Contains("Страница была создана заново"))
{
result = " ''(создана заново)''";
}
else
{
result = " ''(восстановлена)''";
}
}
else
{
result = " ''(не восстановлена)''";
}
}
}
}
filler = "";
for (int i = 0; i < subsection.Level - 1; ++i)
{
filler += "*";
}
titles.Add(filler + " " + subsection.Title.Trim());
}
}
if (titles.Count(s => s.Contains("=")) > 0)
{
titles[0] = "2=<div>" + titles[0].Substring(2);
}
sw.Write(string.Join("\n", titles.ConvertAll(c => c).ToArray()));
if (titles.Count(s => s.Contains("=")) > 0)
{
sw.Write("</div>");
}
sw.Write("}}\n\n");
}
sw.WriteLine("{{/Подвал}}");
}
}
static int Main(string[] args)
{
Wiki wiki = new Wiki("http://ru.wikipedia.org/w/");
wiki.SleepBetweenQueries = 2;
if (string.IsNullOrEmpty(Settings.Default.Login) ||
string.IsNullOrEmpty(Settings.Default.Password))
{
Console.Out.WriteLine("Please add login and password to the configuration file.");
return 0;
}
Console.Out.WriteLine("Logging in as " + Settings.Default.Login + "...");
try
{
WikiCache.Login(wiki, Settings.Default.Login, Settings.Default.Password);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return 0;
}
Console.Out.WriteLine("Logged in as " + Settings.Default.Login + ".");
string listText;
try
{
listText = wiki.LoadText("Шаблон:Список подводящих итоги");
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return -1;
}
StringReader reader = new StringReader(listText);
HashSet<string> users = new HashSet<string>();
Regex userRE = new Regex(@"^\*\s*\[\[(User|Участник):(.+)\|.+\]\]\s*$", RegexOptions.IgnoreCase);
string line;
while ((line = reader.ReadLine()) != null)
{
Match m = userRE.Match(line);
if (m.Success)
{
users.Add(m.Groups[2].Value.ToLower());
}
}
ParameterCollection parameters = new ParameterCollection
{
{ "generator", "embeddedin" },
{ "geititle", "Template:Db-discussion" },
{ "geilimit", "max"},
{ "prop", "info" },
{ "intoken", "delete" },
{ "inprop", "talkid" }
};
Regex templateRE = new Regex(@"\{\{db-discussion\|(\d+)\|(дата|причина)\s*=\s*(.+?)\}\}", RegexOptions.IgnoreCase);
XmlDocument doc;
try
{
doc = wiki.Enumerate(parameters, true);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return -1;
}
bool failed = false;
foreach (XmlNode page in doc.SelectNodes("//page"))
{
string title = page.Attributes["title"].Value;
parameters = new ParameterCollection
{
{ "prop", "revisions" },
{ "rvprop", "content" },
{ "rvlimit", "1" }
};
XmlDocument xml;
try
{
xml = wiki.Query(QueryBy.Titles, parameters, title);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
failed = true;
continue;
}
XmlNode node = xml.SelectSingleNode("//rev");
if (node == null || node.FirstChild == null)
{
failed = true;
continue;
}
string content = node.FirstChild.Value;
Match m = templateRE.Match(content);
if (m.Success)
{
string timestamp = m.Groups[1].Value;
parameters = new ParameterCollection
{
{ "prop", "revisions" },
{ "rvprop", "timestamp|user|content" },
{ "rvlimit", "1" },
{ "rvstart", timestamp },
{ "rvdir", "newer" },
};
try
{
xml = wiki.Query(QueryBy.Titles, parameters, new string[] { title }, 500, false);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
failed = true;
continue;
}
node = xml.SelectSingleNode("//rev");
if (node != null && users.Contains(node.Attributes["user"].Value.ToLower()))
{
content = node.FirstChild.Value;
m = templateRE.Match(content);
if (m.Success && m.Groups[1].Value == timestamp)
{
string comment;
if (m.Groups[2].Value.ToLower() == "причина")
{
comment = m.Groups[3].Value;
}
else
{
DateTime talkDate;
if (DateTime.TryParse(m.Groups[3].Value, null, System.Globalization.DateTimeStyles.AssumeUniversal,
out talkDate))
{
comment = string.Format("согласно итогу обсуждения [[ВП:К удалению/{0}#{1}]]",
talkDate.ToUniversalTime().ToString("d MMMM yyyy"),
title);
}
else
{
continue;
}
}
string reason = string.Format("удалил [[User:{0}|{0}]]: {1}",
node.Attributes["user"].Value,
comment);
string token = page.Attributes["deletetoken"].Value;
try
{
wiki.Delete(title, reason, token);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
failed = true;
continue;
}
parameters = new ParameterCollection
{
{ "list", "backlinks" },
{ "blfilterredir", "redirects" },
{ "bllimit", "max" },
{ "bltitle", title },
};
try
{
xml = wiki.Enumerate(parameters, true);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
failed = true;
continue;
}
foreach (XmlNode backlink in xml.SelectNodes("//bl"))
{
try
{
wiki.Delete(backlink.Attributes["title"].Value,
"[[ВП:КБУ#П1|П1]]: перенаправление в никуда",
wiki.Token);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
failed = true;
continue;
}
}
}
}
}
}
return failed ? -1 : 0;
}
static void Main(string[] args)
{
Wiki wiki = new Wiki("http://ru.wikipedia.org/w/");
if (string.IsNullOrEmpty(Settings.Default.Login) ||
string.IsNullOrEmpty(Settings.Default.Password))
{
Console.Out.WriteLine("Please add login and password to the configuration file.");
return;
}
Console.Out.WriteLine("Logging in as " + Settings.Default.Login + "...");
try
{
wiki.Login(Settings.Default.Login, Settings.Default.Password);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return;
}
Console.Out.WriteLine("Logged in as " + Settings.Default.Login + ".");
DateTime time = new DateTime(2009, 1, 1);
Dictionary<string, DateTime> pages = new Dictionary<string, DateTime>();
pages.Add("Википедия:Сообщения об ошибках", new DateTime());
while (time <= DateTime.Now)
{
string name = time.ToString("MMMM yyyy");
pages.Add("Википедия:Сообщения об ошибках/Архив/" + name, new DateTime());
time = time.AddMonths(1);
}
string timeString = DateTime.Now.ToUniversalTime().ToString("HH:mm, d MMMM yyyy (UTC)");
ParameterCollection parameters = new ParameterCollection();
parameters.Add("prop", "info");
XmlDocument doc;
try
{
doc = wiki.Query(QueryBy.Titles, parameters, pages.Keys);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return;
}
XmlNodeList nodes = doc.SelectNodes("//page");
foreach (XmlNode node in nodes)
{
if (node.Attributes["missing"] != null)
{
continue;
}
string title = node.Attributes["title"].Value;
string timestamp = node.Attributes["touched"].Value;
pages[title] = DateTime.Parse(timestamp, null, DateTimeStyles.AssumeUniversal);
}
bool changed = false;
List<Message> messages = new List<Message>();
Console.Out.WriteLine("Downloading statistic data...");
foreach (string s in pages.Keys)
{
string timestamp = pages[s].ToString("yyyyMMddHHmmss");
string filename = s.Replace(':', '-').Replace('/', '-') + ".bin";
string text = null;
if (File.Exists(filename))
{
using (FileStream fs = new FileStream(filename, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string ts = sr.ReadLine();
if (ts == timestamp)
{
text = sr.ReadToEnd();
}
}
}
if (text == null)
{
using (FileStream fs = new FileStream(filename, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
try
{
text = wiki.LoadText(s);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return;
}
sw.WriteLine(timestamp);
sw.Write(text);
changed = true;
}
}
messages.AddRange(GetMessagesFromPage(text, s.Contains("Архив")));
}
Console.Out.WriteLine("Data downloaded.");
Console.Out.WriteLine("Processing data...");
int newCount = messages.Count(m => m.Opened &&
((m.Archived && m.Closed) || !m.Archived));
int closedCount = messages.Count(m => m.Opened && m.Closed);
using (StreamWriter sw =
new StreamWriter("output.txt"))
{
sw.Write("На этой странице приводится статистика обработки [[ВП:СО|сообщений об ошибках]]");
sw.Write(" по состоянию на " + timeString + ".");
sw.Write(" Всего было подано " + Messages(messages.Count));
sw.Write(", из них не учтено " + Messages(messages.Count - newCount));
sw.Write(", учтено " + Messages(newCount));
sw.Write(" (" + closedCount.ToString() + " распознаны как закрытые).");
sw.Write("{{.ref|<ref>На [[ВП:СО]] учитываются сообщения, которые включают в себя строчку «Автор сообщения» с датой добавления. На страницах архивов учитываются сообщения, которые содержат строчку «Автор сообщения» или «Сообщил» и больше одной даты, при этом новейшая из них считается датой закрытия.</ref>}}\n\n");
PrintWeeks(sw, messages);
PrintMonths(sw, messages);
PrintYears(sw, messages);
PrintPeriods(sw, messages);
sw.WriteLine("== Примечания ==");
sw.WriteLine("{{примечания}}");
}
Console.Out.WriteLine("Results are ready.");
if (changed)
{
Console.Out.WriteLine("Updating the wiki page...");
using (TextReader sr =
new StreamReader("output.txt"))
{
string text = sr.ReadToEnd();
try
{
wiki.Save("User:LEMeZza/Статистика ВП:СО",
text,
"обновление статистики обработки [[ВП:СО|сообщений об ошибках]]");
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return;
}
}
}
else
{
Console.Out.WriteLine("No changes, skipping the update.");
}
wiki.Logout();
Console.Out.WriteLine("Done.");
}
public void Analyze(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", _l10i.Category);
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info");
XmlDocument doc = wiki.Enumerate(parameters, true);
XmlNodeList pages = doc.SelectNodes("//page");
List<Day> days = new List<Day>();
foreach (XmlNode page in pages)
{
string prefix = _l10i.MainPage + "/";
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring(prefix.Length);
Day day = new Day();
try
{
day.Date = DateTime.Parse(date,
_l10i.Culture,
DateTimeStyles.AssumeUniversal);
}
catch (FormatException)
{
continue;
}
string fileName = _cacheDir + date + ".bin";
string text = "";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
Match m = _l10i.closedRE.Match(text);
if (m.Success)
{
Console.Out.WriteLine("Closing " + pageName + "...");
text = _l10i.ClosePage(text);
wiki.Save(pageName,
text,
_l10i.CloseComment);
continue;
}
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
using (StreamWriter sw =
new StreamWriter(_cacheDir + "MainPage.txt"))
{
if (!string.IsNullOrEmpty(_l10i.SectionTitle))
{
sw.WriteLine("== " + _l10i.SectionTitle + " ==\n");
}
sw.WriteLine("{{" + _l10i.TopTemplate + "}}\n");
foreach (Day day in days)
{
sw.Write("{{" + _l10i.Template + "|" + day.Date.ToString("yyyy-M-d") + "|");
List<string> titles = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
RemoveStrikeOut(section);
StrikeOutSection(section);
string result = section.Reduce("", SubsectionsList);
if (result.Length > 0)
{
result = " • <small>" + result.Substring(3) + "</small>";
}
string title;
if (_l10i.Processor != null)
{
title = _l10i.Processor(section).Trim();
}
else
{
title = section.Title.Trim();
}
titles.Add(title + result);
}
sw.Write(string.Join(" • ", titles.ConvertAll(c => c).ToArray()));
sw.Write("}}\n\n");
}
sw.WriteLine("{{" + _l10i.BottomTemplate + "}}");
}
}
static void Main(string[] args)
{
Wiki wiki = new Wiki("http://ru.wikipedia.org/w/");
if (string.IsNullOrEmpty(Settings.Default.Login) ||
string.IsNullOrEmpty(Settings.Default.Password))
{
Console.Out.WriteLine("Please add login and password to the configuration file.");
return;
}
Console.Out.WriteLine("Logging in as " + Settings.Default.Login + "...");
try
{
wiki.Login(Settings.Default.Login, Settings.Default.Password);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return;
}
Console.Out.WriteLine("Logged in as " + Settings.Default.Login + ".");
wiki.SleepBetweenQueries = 3;
Regex re = new Regex(@"\*\s*\[\[User:(.+?)\]\]\s*→\s*\[\[User:(.+?)\]\]");
Dictionary<string, string> renamedUsers = new Dictionary<string, string>();
string renamedUsersData = wiki.LoadText("Википедия:Проект:Патрулирование/Статистика/1k+/Переименования");
using (TextReader sr = new StringReader(renamedUsersData))
{
string line;
while ((line = sr.ReadLine()) != null)
{
Match m = re.Match(line);
if (m.Success)
{
string oldName = m.Groups[1].Value;
string newName = m.Groups[2].Value;
if (!renamedUsers.ContainsKey(oldName))
{
renamedUsers.Add(oldName, newName);
}
}
}
}
DateTime currentMonth = new DateTime(DateTime.Today.Year, DateTime.Today.Month, 1);
DateTime now = currentMonth;
DateTime firstReviewMonth = new DateTime(2008, 9, 1);
while (currentMonth > firstReviewMonth)
{
DateTime previousMonth = currentMonth.AddMonths(-1);
if (File.Exists("output" + previousMonth.ToString("yyyy-MM") + ".txt"))
{
currentMonth = currentMonth.AddMonths(-1);
continue;
}
string start = previousMonth.ToString("yyyy-MM-ddTHH:mm:ssZ");
string stop = currentMonth.ToString("yyyy-MM-ddTHH:mm:ssZ");
Console.Out.WriteLine("Quering list of editors for " + previousMonth.ToString("MMMM yyyy") + "...");
ParameterCollection parameters = new ParameterCollection();
parameters.Add("list", "logevents");
parameters.Add("letype", "review");
parameters.Add("lestart", start);
parameters.Add("leend", stop);
parameters.Add("ledir", "newer");
parameters.Add("lelimit", "max");
Dictionary<string, User> users = new Dictionary<string, User>();
XmlNode continueNode = null;
while (true)
{
XmlDocument doc;
try
{
doc = wiki.MakeRequest(Claymore.SharpMediaWiki.Action.Query, parameters);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return;
}
continueNode = doc.SelectSingleNode("//query-continue");
if (continueNode != null)
{
string name = continueNode.FirstChild.Attributes[0].Name;
string value = continueNode.FirstChild.Attributes[0].Value;
parameters.Set(name, value);
}
XmlNodeList entries = doc.SelectNodes("//item[@action!=\"approve-a\" and @action!=\"approve-ia\"]");
foreach (XmlNode entry in entries)
{
string username = renamedUsers.ContainsKey(entry.Attributes["user"].Value)
? renamedUsers[entry.Attributes["user"].Value]
: entry.Attributes["user"].Value;
string ns = entry.Attributes["ns"].Value;
if (!users.ContainsKey(username))
{
User user = new User(username);
if (ns == "0")
{
user.ArticleActions = 1;
}
else if (ns == "14")
{
user.CategoryActions = 1;
}
else if (ns == "10")
{
user.TemplateActions = 1;
}
else if (ns == "6")
{
user.FileActions = 1;
}
users.Add(username, user);
}
else
{
User user = users[username];
if (ns == "0")
{
++user.ArticleActions;
}
else if (ns == "14")
{
++user.CategoryActions;
}
else if (ns == "10")
{
++user.TemplateActions;
}
else if (ns == "6")
{
++user.FileActions;
}
users[username] = user;
}
}
if (continueNode == null)
{
break;
}
}
Console.Out.WriteLine("Processing data...");
List<User> userList = new List<User>(users.Select(s => s.Value));
userList.Sort(CompareUsers);
using (StreamWriter sw =
new StreamWriter("output" + previousMonth.ToString("yyyy-MM") + ".txt", false))
{
int totalActions = 0;
int totalArticleActions = 0;
int totalCategoryActions = 0;
int totalTemplateActions = 0;
int totalFileActions = 0;
bool bots = false;
sw.WriteLine("== " + previousMonth.ToString("MMMM") + " ==");
sw.WriteLine("{| class=\"standard sortable\"");
sw.WriteLine("!№!!Участник!!всего!!статей!!категорий!!шаблонов!!файлов");
for (int i = 0, j = 1; i < userList.Count; ++i)
{
if (userList[i].Name.Contains("Lockalbot") ||
userList[i].Name.Contains("Secretary"))
{
bots = true;
continue;
}
sw.WriteLine("|-");
totalActions += userList[i].Actions;
totalArticleActions += userList[i].ArticleActions;
totalCategoryActions += userList[i].CategoryActions;
totalTemplateActions += userList[i].TemplateActions;
totalFileActions += userList[i].FileActions;
string line = string.Format("|{0}||[[User:{1}|]]||{2}||{3}||{4}||{5}||{6}",
j++,
userList[i].Name,
userList[i].Actions,
userList[i].ArticleActions,
userList[i].CategoryActions,
userList[i].TemplateActions,
userList[i].FileActions);
sw.WriteLine(line);
}
sw.WriteLine("|-");
sw.WriteLine("|||Итого||{0}||{1}||{2}||{3}||{4}",
totalActions,
totalArticleActions,
totalCategoryActions,
totalTemplateActions,
totalFileActions);
sw.WriteLine("|}");
if (bots)
{
sw.WriteLine("; Боты");
sw.WriteLine("{| class=\"standard sortable\"");
sw.WriteLine("!№!!Участник!!всего!!статей!!категорий!!шаблонов!!файлов");
for (int i = 0, j = 1; i < userList.Count; ++i)
{
if (userList[i].Name.Contains("Lockalbot") ||
userList[i].Name.Contains("Secretary"))
{
sw.WriteLine("|-");
string line = string.Format("|{0}||[[User:{1}|]]||{2}||{3}||{4}||{5}||{6}",
j++,
userList[i].Name,
userList[i].Actions,
userList[i].ArticleActions,
userList[i].CategoryActions,
userList[i].TemplateActions,
userList[i].FileActions);
sw.WriteLine(line);
}
}
sw.WriteLine("|}");
}
sw.WriteLine("\n— ~~~~");
}
currentMonth = currentMonth.AddMonths(-1);
}
currentMonth = new DateTime(2008, 9, 1);
Dictionary<string, List<MonthStat>> userStatistics = new Dictionary<string, List<MonthStat>>();
while (currentMonth < now)
{
using (TextReader sr =
new StreamReader("output" + currentMonth.ToString("yyyy-MM") + ".txt"))
{
string line;
while ((line = sr.ReadLine()) != null)
{
if (line == "|-")
{
line = sr.ReadLine();
string[] fields = line.Split(new string[] { "||" }, StringSplitOptions.RemoveEmptyEntries);
string user = fields[1];
int actions = int.Parse(fields[2]);
if (user.StartsWith("[[User:"******"output" + currentMonth.ToString("yyyy-MM") + ".txt"))
{
string line;
while ((line = sr.ReadLine()) != null)
{
if (line == "|-")
{
line = sr.ReadLine();
string[] fields = line.Split(new string[] { "||" }, StringSplitOptions.RemoveEmptyEntries);
string user = fields[1];
int actions = int.Parse(fields[2]);
if (userStatistics.ContainsKey(user))
{
List<MonthStat> stats = userStatistics[user];
stats.Add(new MonthStat(currentMonth, actions));
}
}
}
}
currentMonth = currentMonth.AddMonths(1);
}
using (StreamWriter sw =
new StreamWriter("output.txt", false))
{
sw.WriteLine("== Статистика ==");
for (int year = DateTime.Today.Year; year >= 2008; --year)
{
DateTime currentYear = new DateTime(year, 1, 1);
List<UserStat> userStats = new List<UserStat>();
foreach (var user in userStatistics)
{
var stats = user.Value.Where(s => (s.Month >= currentYear && s.Month < currentYear.AddYears(1)));
UserStat userStat = new UserStat(user.Key, stats);
userStat.enterDate = user.Value.Where(s => s.Actions >= 1000).Min(s => s.Month);
userStat.ActionsBefore = user.Value.Where(s => s.Month < currentYear).Sum(s => s.Actions);
userStat.Max = user.Value.Max(s => s.Actions);
if (stats.Count() > 0 && userStat.enterDate.Year <= year)
{
userStats.Add(userStat);
}
}
userStats.Sort(CompareUserStat);
sw.WriteLine("\n=== {0} ===", currentYear.Year);
sw.WriteLine("{| class=\"wikitable sortable\"");
sw.Write("! № !! Участник");
currentMonth = currentYear;
while (currentMonth < currentYear.AddYears(1))
{
sw.Write(" !! " + currentMonth.ToString("MMM yy"));
currentMonth = currentMonth.AddMonths(1);
}
sw.Write(" !! За {0} !! На конец {0} ", currentYear.Year);
sw.WriteLine();
for (int index = 0; index < userStats.Count; ++index)
{
sw.WriteLine("|-");
sw.WriteLine(userStats[index].IsBot ? "! Бот" : string.Format("! {0}", index + 1));
sw.Write(string.Format("| {0}", userStats[index].Name));
int max = userStats[index].Stat.Max(s => s.Actions);
currentMonth = currentYear;
int totalActions = 0;
while (currentMonth < currentYear.AddYears(1))
{
int actions = 0;
foreach (var item in userStats[index].Stat)
{
if (item.Month == currentMonth)
{
actions = item.Actions;
break;
}
}
if (actions != userStats[index].Max)
{
sw.Write(" || " + actions);
}
else
{
sw.Write(string.Format(" || '''{0}'''", actions));
}
totalActions += actions;
currentMonth = currentMonth.AddMonths(1);
}
sw.Write(string.Format(" || {0}", totalActions));
sw.Write(string.Format(" || {0}", totalActions + userStats[index].ActionsBefore));
sw.WriteLine();
}
sw.WriteLine("|}");
}
}
currentMonth = new DateTime(now.Year, now.Month, 1).AddMonths(-1);
Console.Out.WriteLine("Updating the wiki page...");
using (TextReader sr =
new StreamReader("output" + currentMonth.ToString("yyyy-MM") + ".txt"))
{
DateTime previousMonth = new DateTime(now.Year, now.Month, 1).AddMonths(-1);
string text = sr.ReadToEnd();
string period = previousMonth.ToString("MMMM yyyy");
wiki.Save(previousMonth.ToString("Википедия:Проект:Патрулирование\\/Статистика\\/yyyy\\/MM"),
text,
"статистика патрулирования за " + period[0].ToString().ToLower() + period.Substring(1));
}
Console.Out.WriteLine("Updating Википедия:Проект:Патрулирование/Статистика/1k+...");
using (TextReader sr =
new StreamReader("output.txt"))
{
string text = sr.ReadToEnd();
wiki.SaveSection("Википедия:Проект:Патрулирование/Статистика/1k+",
"1",
text,
"обновление");
}
wiki.Save(currentMonth.ToString("Википедия:Проект:Патрулирование\\/Статистика\\/yyyy"),
string.Format("#REDIRECT [[{0}]]", currentMonth.ToString("Википедия:Проект:Патрулирование\\/Статистика\\/yyyy\\/MM")),
"обновление");
Console.Out.WriteLine("Done.");
wiki.Logout();
}
public virtual string GetData(Wiki wiki)
{
WebClient client = new WebClient();
foreach (var category in Categories)
{
Cache.LoadPageList(client, category, Module.Language, Depth, Hours);
}
foreach (var category in CategoriesToIgnore)
{
Cache.LoadPageList(client, category, Module.Language, Depth, Hours);
}
try
{
return wiki.LoadText(Page);
}
catch (WikiPageNotFound)
{
return "";
}
}
public void Load(Wiki wiki)
{
Parse(wiki.LoadText(Title));
}
public void Analyze(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения переименования страниц");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info|revisions");
parameters.Add("rvprop", "timestamp");
XmlDocument doc = wiki.Enumerate(parameters, true);
XmlNodeList pages = doc.SelectNodes("//page");
List<Day> days = new List<Day>();
DateTime start = DateTime.Today;
Regex closedRE = new Regex(@"(\{{2}ВПКПМ-Навигация\}{2}\s*\{{2}(Закрыто|Closed|закрыто|closed)\}{2})|(\{{2}(Закрыто|Closed|закрыто|closed)\}{2}\s*\{{2}ВПКПМ-Навигация\}{2})");
wiki.SleepBetweenEdits = 10;
wiki.SleepBetweenQueries = 2;
DateTime cutOffDate = new DateTime(2009, 3, 21);
foreach (XmlNode page in pages)
{
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring("Википедия:К переименованию/".Length);
Day day = new Day();
try
{
day.Date = DateTime.Parse(date,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
catch (FormatException)
{
continue;
}
string fileName = _cacheDir + date + ".bin";
string text = "";
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
DateTime lastEdit = DateTime.Parse(page.FirstChild.FirstChild.Attributes["timestamp"].Value, null, DateTimeStyles.AssumeUniversal);
Match m = closedRE.Match(text);
if ((DateTime.Now - lastEdit).TotalDays > 2 && m.Success)
{
text = text.Replace("{{ВПКПМ-Навигация}}", "{{ВПКПМ-Навигация|nocat=1}}");
try
{
string revid = wiki.Save(pageName,
text,
"обсуждение закрыто");
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(revid);
sw.Write(text);
}
}
catch (WikiException)
{
}
continue;
}
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
using (StreamWriter sw =
new StreamWriter(_cacheDir + "MainPage.txt"))
{
sw.WriteLine("{{/Шапка}}\n");
sw.WriteLine("{{Переименование статей/Статьи, вынесенные на переименование}}\n");
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
foreach (Day day in days)
{
Console.Out.WriteLine("Analyzing " + day.Date.ToString("d MMMM yyyy") + "...");
sw.Write("{{Переименование статей/День|" + day.Date.ToString("yyyy-M-d") + "|\n");
List<string> titles = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
string filler = "";
string result = "";
RemoveStrikeOut(section);
StrikeOutSection(section);
if (section.SectionText.ToLower().Contains("{{mark out}}"))
{
section.Title = "{{mark out|" + section.Title.Trim() + "}}";
}
bool hasVerdict = section.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") > 0;
bool hasAutoVerdict = section.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") > 0;
if (hasVerdict || hasAutoVerdict || section.Title.Contains("<s>"))
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success && !m.Groups[1].Value.StartsWith(":Категория:"))
{
string link = m.Groups[1].Value;
string movedTo;
bool moved = MovedTo(wiki, link, day.Date, out movedTo);
if (moved && string.IsNullOrEmpty(movedTo))
{
result = " ''(переименовано)''";
}
else if (moved)
{
result = string.Format(" ''({1}переименовано в «[[{0}]]»)''",
movedTo.StartsWith("Файл:") ? ":" + movedTo : movedTo, hasAutoVerdict ? "де-факто " : "");
}
else
{
result = " ''(не переименовано)''";
}
}
}
for (int i = 0; i < section.Level - 1; ++i)
{
filler += "*";
}
titles.Add(filler + " " + section.Title.Trim() + result);
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
if (subsection.SectionText.ToLower().Contains("{{mark out}}"))
{
subsection.Title = "{{mark out|" + subsection.Title.Trim() + "}}";
}
result = "";
hasVerdict = subsection.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") > 0;
hasAutoVerdict = subsection.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") > 0;
if (hasVerdict || hasAutoVerdict || subsection.Title.Contains("<s>"))
{
Match m = wikiLinkRE.Match(subsection.Title);
if (m.Success && !m.Groups[1].Value.StartsWith(":Категория:"))
{
string link = m.Groups[1].Value;
string movedTo;
bool moved = MovedTo(wiki, link, day.Date, out movedTo);
if (moved && string.IsNullOrEmpty(movedTo))
{
result = " ''(переименовано)''";
}
else if (moved)
{
result = string.Format(" ''({1}переименовано в «[[{0}]]»)''",
movedTo.StartsWith("Файл:") ? ":" + movedTo : movedTo, hasAutoVerdict ? "де-факто " : "");
}
else
{
result = " ''(не переименовано)''";
}
}
}
filler = "";
for (int i = 0; i < subsection.Level - 1; ++i)
{
filler += "*";
}
titles.Add(filler + " " + subsection.Title.Trim() + result);
}
}
if (titles.Count(s => s.Contains("=")) > 0)
{
titles[0] = "2=<li>" + titles[0].Substring(2) + "</li>";
}
sw.Write(string.Join("\n", titles.ConvertAll(c => c).ToArray()));
sw.Write("}}\n\n");
}
sw.WriteLine("|}\n");
sw.WriteLine("{{/Окончание}}");
}
}
static void Main(string[] args)
{
Wiki wiki = new Wiki("http://ru.wikipedia.org/w/");
wiki.SleepBetweenQueries = 2;
if (string.IsNullOrEmpty(Settings.Default.Login) ||
string.IsNullOrEmpty(Settings.Default.Password))
{
Console.Out.WriteLine("Please add login and password to the configuration file.");
return;
}
Console.Out.WriteLine("Logging in as " + Settings.Default.Login + "...");
try
{
WikiCache.Login(wiki, Settings.Default.Login, Settings.Default.Password);
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return;
}
Console.Out.WriteLine("Logged in as " + wiki.User + ".");
Dictionary<string, int> requestNumbers = new Dictionary<string, int>();
string text = wiki.LoadText("Википедия:Заявки на арбитраж/Все страницы");
StringReader reader = new StringReader(text);
Regex re = new Regex(@" \[\[ВП:(\d+)\]\] — \[\[(.+?)\|");
string line;
while ((line = reader.ReadLine()) != null)
{
Match m = re.Match(line);
if (m.Success)
{
int number = int.Parse(m.Groups[1].Value);
string page = m.Groups[2].Value;
requestNumbers.Add(page, number);
}
}
var parameters = new ParameterCollection
{
{"list" , "categorymembers"},
{"cmtitle", "Категория:Википедия:Заявки в Арбитражный комитет"},
{"cmlimit", "max"},
};
List<ArbComEntry> entries = new List<ArbComEntry>();
var doc = wiki.Enumerate(parameters, true);
foreach (XmlNode page in doc.SelectNodes("//cm"))
{
parameters = new ParameterCollection
{
{"list" , "backlinks"},
{"bltitle", page.Attributes["title"].Value},
{"bllimit", "max"},
{"blfilterredir", "redirects"},
};
int number = int.MinValue;
if (requestNumbers.ContainsKey(page.Attributes["title"].Value))
{
number = requestNumbers[page.Attributes["title"].Value];
}
else
{
XmlDocument xml = wiki.Enumerate(parameters, true);
foreach (XmlNode link in xml.SelectNodes("//bl"))
{
if (link.Attributes["title"].Value.StartsWith("Википедия:"))
{
if (int.TryParse(link.Attributes["title"].Value.Replace("Википедия:", ""), out number))
{
break;
}
else
{
number = int.MinValue;
}
}
}
}
ArbComEntry entry = new ArbComEntry();
entry.Number = number;
entry.Page = page.Attributes["title"].Value;
entries.Add(entry);
}
string prefix = "Википедия:Заявки на арбитраж/";
StringBuilder sb = new StringBuilder("== Заявки ==\n");
entries.Sort(CompareEntries);
foreach (ArbComEntry entry in entries)
{
string requestNumber = "";
if (entry.Number != int.MinValue)
{
requestNumber = string.Format("[[ВП:{0}]] — ", entry.Number);
}
string title = entry.Page.Substring(prefix.Length);
sb.AppendFormat("* {2}[[{1}{0}|{0}]] [[Обсуждение Википедии:Заявки на арбитраж/{0}|±]] ([[{1}{0}/Дискуссия арбитров|д]])\n", title, prefix, requestNumber);
}
text = sb.ToString();
wiki.SaveSection("Википедия:Заявки на арбитраж/Все страницы", "1", text, "/* Заявки */ обновление");
}
public void UpdateArchivePages(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения объединения страниц");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info");
XmlDocument doc = wiki.Enumerate(parameters, true);
XmlNodeList pages = doc.SelectNodes("//page");
DateTime minDate = DateTime.Now;
foreach (XmlNode page in pages)
{
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring("Википедия:К объединению/".Length);
try
{
DateTime day = DateTime.Parse(date,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
if (day < minDate)
{
minDate = day;
}
}
catch (FormatException)
{
continue;
}
}
List<string> titles = new List<string>();
minDate = new DateTime(minDate.Year, minDate.Month, 1);
DateTime currentMonth = new DateTime(DateTime.Today.Year,
DateTime.Today.Month, 1);
DateTime start = minDate;
while (start <= currentMonth)
{
string date = start.ToString("yyyy-MM");
string pageName = "Википедия:Архив запросов на объединение/" + date;
titles.Add(pageName);
start = start.AddMonths(1);
}
parameters.Clear();
parameters.Add("prop", "info");
XmlDocument archivesDoc = wiki.Query(QueryBy.Titles, parameters, titles);
pages = archivesDoc.SelectNodes("//page");
foreach (XmlNode archivePage in pages)
{
string archiveName = archivePage.Attributes["title"].Value;
string date = archiveName.Substring("Википедия:Архив запросов на объединение/".Length);
DateTime archiveDate;
try
{
archiveDate = DateTime.Parse(date,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
catch (FormatException)
{
continue;
}
string fileName = _cacheDir + "Archive-" + date + ".txt";
start = archiveDate;
DateTime end = start.AddMonths(1);
titles.Clear();
while (start < end)
{
string pageDate = start.ToString("d MMMM yyyy",
CultureInfo.CreateSpecificCulture("ru-RU"));
string prefix = "Википедия:К объединению/";
string pageName = prefix + pageDate;
titles.Add(pageName);
start = start.AddDays(1);
}
parameters.Clear();
parameters.Add("prop", "info");
List<Day> days = new List<Day>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles);
XmlNodeList archives = xml.SelectNodes("//page");
foreach (XmlNode page in archives)
{
string pageName = page.Attributes["title"].Value;
string dateString = pageName.Substring("Википедия:К объединению/".Length);
string pageFileName = _cacheDir + dateString + ".bin";
Day day = new Day();
day.Archived = doc.SelectSingleNode("//page[@title=\"" + pageName + "\"]") == null;
try
{
day.Date = DateTime.Parse(dateString,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
catch (FormatException)
{
continue;
}
if (page.Attributes["missing"] != null)
{
day.Exists = false;
days.Add(day);
continue;
}
string text = LoadPageFromCache(pageFileName,
page.Attributes["lastrevid"].Value, pageName);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
CachePage(pageFileName, page.Attributes["lastrevid"].Value, text);
}
day.Exists = true;
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
StringBuilder textBuilder = new StringBuilder();
textBuilder.AppendLine("{{Навигация по архиву КОБ}}\n{{Объединение статей/Начало}}");
StringBuilder sb = new StringBuilder();
foreach (Day day in days)
{
sb.Append("{{Объединение статей/День|" + day.Date.ToString("yyyy-M-d") + "|\n");
if (!day.Archived && day.Exists)
{
sb.Append("''обсуждение не завершено''}}\n\n");
continue;
}
if (!day.Exists)
{
sb.Append("''нет обсуждений''}}\n\n");
continue;
}
List<string> sectionTitles = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
string filler = "";
for (int i = 0; i < section.Level - 1; ++i)
{
filler += "*";
}
sectionTitles.Add(filler + " " + section.Title.Trim());
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
filler = "";
for (int i = 0; i < subsection.Level - 1; ++i)
{
filler += "*";
}
sectionTitles.Add(filler + " " + subsection.Title.Trim());
}
}
if (sectionTitles.Count(s => s.Contains("=")) > 0)
{
titles[0] = "2=<li>" + titles[0].Substring(2) + "</li>";
}
sb.Append(string.Join("\n", sectionTitles.ConvertAll(c => c).ToArray()));
sb.Append("}}\n\n");
}
sb.Replace("<s>", "");
sb.Replace("</s>", "");
sb.Replace("<strike>", "");
sb.Replace("</strike>", "");
textBuilder.Append(sb.ToString());
textBuilder.AppendLine("{{Объединение статей/Конец}}");
if (File.Exists(fileName))
{
using (TextReader sr = new StreamReader(fileName))
{
string text = sr.ReadToEnd();
if (text == textBuilder.ToString())
{
continue;
}
}
}
Console.Out.WriteLine("Updating " + archiveName + "...");
wiki.Save(archiveName,
textBuilder.ToString(),
"обновление");
using (StreamWriter sw =
new StreamWriter(fileName))
{
sw.Write(textBuilder.ToString());
}
}
}
public void Run(Wiki wiki)
{
Directory.CreateDirectory(_cacheDir);
Console.Out.WriteLine("Updating " + _page);
Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}");
Regex timeRE = new Regex(@"(\d{2}:\d{2}\, \d\d? [а-я]+ \d{4}) \(UTC\)");
ParameterCollection parameters = new ParameterCollection();
parameters.Add("prop", "info|revisions");
parameters.Add("intoken", "edit");
XmlDocument doc = wiki.Query(QueryBy.Titles, parameters, _page);
XmlNode page = doc.SelectSingleNode("//page");
string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
string pageName = page.Attributes["title"].Value;
string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value;
string editToken = page.Attributes["edittoken"].Value;
string starttimestamp = queryTimestamp;
string text = "";
string fileName = _cacheDir + WikiCache.EscapePath(_page);
if (File.Exists(fileName))
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress))
using (TextReader sr = new StreamReader(gs))
{
string revid = sr.ReadLine();
if (revid == page.Attributes["lastrevid"].Value)
{
Console.Out.WriteLine("Loading " + pageName + "...");
text = sr.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
starttimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(page.Attributes["lastrevid"].Value);
sw.Write(text);
}
}
Dictionary<string, List<WikiPageSection>> titles = new Dictionary<string, List<WikiPageSection>>();
WikiPage wikiPage = WikiPage.Parse(pageName, text);
foreach (WikiPageSection section in wikiPage.Sections)
{
if (section.Subsections.Count(s => s.Title.ToLower(CultureInfo.CreateSpecificCulture("ru-RU")).Trim() == "итог") == 0)
{
Match m = wikiLinkRE.Match(section.Title);
if (m.Success)
{
string title = m.Groups[1].Value.Trim();
if (titles.ContainsKey(title))
{
titles[title].Add(section);
}
else
{
titles.Add(title, new List<WikiPageSection>());
titles[title].Add(section);
}
}
}
}
parameters.Clear();
parameters.Add("prop", "info");
Dictionary<string, string> normalizedTitles = new Dictionary<string, string>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles.Keys);
foreach (XmlNode node in xml.SelectNodes("//n"))
{
normalizedTitles.Add(node.Attributes["to"].Value,
node.Attributes["from"].Value);
}
List<string> notificationList = new List<string>();
XmlNodeList missingTitles = xml.SelectNodes("//page");
foreach (XmlNode node in missingTitles)
{
string title = node.Attributes["title"].Value;
IEnumerable<WikiPageSection> sections;
if (titles.ContainsKey(title))
{
sections = titles[title];
}
else
{
sections = titles[normalizedTitles[title]];
}
if (node.Attributes["missing"] != null)
{
parameters.Clear();
parameters.Add("list", "logevents");
parameters.Add("letype", "delete");
parameters.Add("lemlimit", "max");
parameters.Add("ledir", "newer");
parameters.Add("letitle", title);
parameters.Add("leprop", "comment|type|user|timestamp");
XmlDocument log = wiki.Enumerate(parameters, true);
XmlNodeList items = log.SelectNodes("//item");
List<DeleteLogEvent> events = new List<DeleteLogEvent>();
foreach (XmlNode item in items)
{
DeleteLogEvent ev = new DeleteLogEvent();
ev.Comment = item.Attributes["comment"].Value;
ev.Deleted = item.Attributes["action"].Value == "delete";
ev.User = item.Attributes["user"].Value;
ev.Timestamp = DateTime.Parse(item.Attributes["timestamp"].Value,
null,
DateTimeStyles.AssumeUniversal);
events.Add(ev);
}
events.Sort(CompareDeleteLogEvents);
if (events.Count > 0 && events[0].Deleted)
{
string comment = FilterWikiMarkup(events[0].Comment);
string message = string.Format("Страница была удалена {1} администратором [[User:{0}|]]. Была указана следующая причина: «{2}». Данное сообщение было автоматически сгенерировано ботом ~~~~.\n",
events[0].User,
events[0].Timestamp.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)",
CultureInfo.CreateSpecificCulture("ru-RU")),
comment);
var pageSections = titles.ContainsKey(title)
? titles[title]
: titles[normalizedTitles[title]];
foreach (WikiPageSection section in pageSections)
{
WikiPageSection verdict = new WikiPageSection(" Итог ",
section.Level + 1,
message);
section.AddSubsection(verdict);
}
}
}
}
string newText = wikiPage.Text;
if (newText.Trim() == text.Trim())
{
return;
}
Console.Out.WriteLine("Updating " + pageName + "...");
string id = wiki.Save(pageName,
"",
newText,
"автоматическое подведение итогов",
MinorFlags.Minor,
CreateFlags.NoCreate,
WatchFlags.None,
SaveFlags.Replace,
true,
basetimestamp,
"",
editToken);
using (FileStream fs = new FileStream(fileName, FileMode.Create))
using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress))
using (StreamWriter sw = new StreamWriter(gs))
{
sw.WriteLine(id);
sw.Write(newText);
}
}
public void Analyze(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения объединения страниц");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info");
XmlDocument doc = wiki.Enumerate(parameters, true);
XmlNodeList pages = doc.SelectNodes("//page");
List<Day> days = new List<Day>();
DateTime start = DateTime.Today;
foreach (XmlNode page in pages)
{
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring("Википедия:К объединению/".Length);
Day day = new Day();
try
{
day.Date = DateTime.Parse(date,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal);
}
catch (FormatException)
{
continue;
}
string fileName = _cacheDir + date + ".bin";
string text = LoadPageFromCache(fileName,
page.Attributes["lastrevid"].Value, pageName);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
CachePage(fileName, page.Attributes["lastrevid"].Value, text);
}
Match m = _closedRE.Match(text);
if (m.Success)
{
Console.Out.WriteLine("Closing " + pageName + "...");
text = text.Replace("{{ВПКОБ-Навигация}}", "{{ВПКОБ-Навигация|nocat=1}}");
wiki.Save(pageName, text, "обсуждение закрыто");
continue;
}
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
using (StreamWriter sw =
new StreamWriter(_cacheDir + "MainPage.txt"))
{
sw.WriteLine("== Текущие обсуждения ==\n");
sw.WriteLine("{{Объединение статей/Статьи, вынесенные для объединения}}\n");
foreach (Day day in days)
{
sw.Write("{{Объединение статей/День|" + day.Date.ToString("yyyy-M-d") + "|\n");
List<string> titles = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
string filler = "";
RemoveStrikeOut(section);
StrikeOutSection(section);
for (int i = 0; i < section.Level - 1; ++i)
{
filler += "*";
}
titles.Add(filler + " " + section.Title.Trim());
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
filler = "";
for (int i = 0; i < subsection.Level - 1; ++i)
{
filler += "*";
}
titles.Add(filler + " " + subsection.Title.Trim());
}
}
if (titles.Count(s => s.Contains("=")) > 0)
{
titles[0] = "2=<li>" + titles[0].Substring(2) + "</li>";
}
sw.Write(string.Join("\n", titles.ConvertAll(c => c).ToArray()));
sw.Write("}}\n\n");
}
sw.WriteLine("{{/Подвал}}");
}
}
public void UpdateArchivePages(Wiki wiki)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "categorymembers");
parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения категорий");
parameters.Add("gcmlimit", "max");
parameters.Add("gcmnamespace", "4");
parameters.Add("prop", "info");
XmlDocument doc = wiki.Enumerate(parameters, true);
XmlNodeList pages = doc.SelectNodes("//page");
DateTime minDate = DateTime.Now;
foreach (XmlNode page in pages)
{
string pageName = page.Attributes["title"].Value;
string date = pageName.Substring("Википедия:Обсуждение категорий/".Length);
DateTime day;
if (DateTime.TryParse(date,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal, out day))
{
if (day < minDate)
{
minDate = day;
}
}
else
{
continue;
}
}
List<string> titles = new List<string>();
minDate = new DateTime(minDate.Year, minDate.Month, 1);
DateTime currentMonth = new DateTime(DateTime.Today.Year,
DateTime.Today.Month, 1);
DateTime start = new DateTime(minDate.Year, 1, 1);
while (start <= currentMonth)
{
string date = start.ToString("yyyy");
string pageName = "Википедия:Обсуждение категорий/Архив/" + date;
titles.Add(pageName);
start = start.AddYears(1);
}
parameters.Clear();
parameters.Add("prop", "info");
XmlDocument archivesDoc = wiki.Query(QueryBy.Titles, parameters, titles);
pages = archivesDoc.SelectNodes("//page");
foreach (XmlNode archivePage in pages)
{
string archiveName = archivePage.Attributes["title"].Value;
string date = archiveName.Substring("Википедия:Обсуждение категорий/Архив/".Length);
int year;
if (!int.TryParse(date, out year))
{
continue;
}
DateTime archiveDate = new DateTime(year, 1, 1);
string fileName = _cacheDir + "Archive-" + date + ".txt";
start = archiveDate;
DateTime end = start.AddYears(1);
titles.Clear();
while (start < end)
{
string pageDate = start.ToString("MMMM yyyy",
CultureInfo.CreateSpecificCulture("ru-RU"));
string prefix = "Википедия:Обсуждение категорий/";
string pageName = prefix + pageDate;
titles.Add(pageName);
start = start.AddMonths(1);
}
parameters.Clear();
parameters.Add("prop", "info");
List<Day> days = new List<Day>();
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles);
XmlNodeList archives = xml.SelectNodes("//page");
foreach (XmlNode page in archives)
{
string pageName = page.Attributes["title"].Value;
string dateString = pageName.Substring("Википедия:Обсуждение категорий/".Length);
string pageFileName = _cacheDir + dateString + ".bin";
Day day = new Day();
if (!DateTime.TryParse(dateString,
CultureInfo.CreateSpecificCulture("ru-RU"),
DateTimeStyles.AssumeUniversal, out day.Date))
{
continue;
}
if (page.Attributes["missing"] != null)
{
continue;
}
string text = LoadPageFromCache(pageFileName,
page.Attributes["lastrevid"].Value, pageName);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
CachePage(pageFileName, page.Attributes["lastrevid"].Value, text);
}
day.Page = WikiPage.Parse(pageName, text);
days.Add(day);
}
days.Sort(CompareDays);
StringBuilder sb = new StringBuilder();
sb.AppendLine("{{Википедия:Обсуждение категорий/Обсуждаемые категории}}\n");
foreach (Day day in days)
{
sb.Append("{{Википедия:Обсуждение категорий/Месяц|" + day.Date.ToString("yyyy-M") + "|\n");
List<string> sectionTitles = new List<string>();
foreach (WikiPageSection section in day.Page.Sections)
{
string filler = "";
for (int i = 0; i < section.Level - 1; ++i)
{
filler += "*";
}
sectionTitles.Add(filler + " " + section.Title.Trim());
List<WikiPageSection> sections = new List<WikiPageSection>();
section.Reduce(sections, SubsectionsList);
foreach (WikiPageSection subsection in sections)
{
filler = "";
for (int i = 0; i < subsection.Level - 1; ++i)
{
filler += "*";
}
sectionTitles.Add(filler + " " + subsection.Title.Trim());
}
}
if (sectionTitles.Count(s => s.Contains("=")) > 0)
{
sectionTitles[0] = "2=<div>" + sectionTitles[0].Substring(2);
}
sb.Append(string.Join("\n", sectionTitles.ConvertAll(c => c).ToArray()));
if (sectionTitles.Count(s => s.Contains("=")) > 0)
{
sb.Append("</div>");
}
sb.Append("}}\n\n");
}
sb.Append("|}");
sb.Replace("<s>", "");
sb.Replace("</s>", "");
sb.Replace("<strike>", "");
sb.Replace("</strike>", "");
if (File.Exists(fileName))
{
using (TextReader sr = new StreamReader(fileName))
{
string txt = sr.ReadToEnd();
if (txt == sb.ToString())
{
continue;
}
}
}
Console.Out.WriteLine("Updating " + archiveName + "...");
wiki.Save(archiveName,
sb.ToString(),
"обновление");
using (StreamWriter sw =
new StreamWriter(fileName))
{
sw.Write(sb.ToString());
}
}
}