private static List<WikiPageSection> SubsectionsList(WikiPageSection section, List<WikiPageSection> aggregator) { Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}"); Match m = wikiLinkRE.Match(section.Title); if (m.Success) { aggregator.Add(section); } return section.Reduce(aggregator, SubsectionsList); }
private string SubsectionsList(WikiPageSection section, string aggregator) { Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}"); Match m = wikiLinkRE.Match(section.Title); if (m.Success) { if (_l10i.Processor != null) { aggregator = aggregator + " • " + _l10i.Processor(section).Trim(); } else { aggregator = aggregator + " • " + section.Title.Trim(); } } aggregator = section.Reduce(aggregator, SubsectionsList); return aggregator; }
public void UpdatePages(Wiki wiki) { Regex closedRE = new Regex(@"(\{{2}ВПКПМ-Навигация\}{2}\s*\{{2}(Закрыто|Closed|закрыто|closed)\}{2})|(\{{2}(Закрыто|Closed|закрыто|closed)\}{2}\s*\{{2}ВПКПМ-Навигация\}{2})"); Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}"); Regex timeRE = new Regex(@"(\d{2}:\d{2}\, \d\d? [а-я]+ \d{4}) \(UTC\)"); ParameterCollection parameters = new ParameterCollection(); parameters.Add("generator", "categorymembers"); parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения переименования страниц"); parameters.Add("gcmlimit", "max"); parameters.Add("gcmnamespace", "4"); parameters.Add("prop", "info|revisions"); parameters.Add("intoken", "edit"); XmlDocument doc = wiki.Enumerate(parameters, true); string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"); XmlNodeList pages = doc.SelectNodes("//page"); foreach (XmlNode page in pages) { int results = 0; string pageName = page.Attributes["title"].Value; string date = pageName.Substring("Википедия:К переименованию/".Length); string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value; string editToken = page.Attributes["edittoken"].Value; Day day = new Day(); if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture("ru-RU"), DateTimeStyles.AssumeUniversal, out day.Date)) { continue; } string fileName = _cacheDir + date + ".bin"; string text = ""; if (File.Exists(fileName)) { using (FileStream fs = new FileStream(fileName, FileMode.Open)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress)) using (TextReader sr = new StreamReader(gs)) { string revid = sr.ReadLine(); if (revid == page.Attributes["lastrevid"].Value) { Console.Out.WriteLine("Loading " + pageName + "..."); text = sr.ReadToEnd(); } } } if (string.IsNullOrEmpty(text)) { Console.Out.WriteLine("Downloading " + pageName + "..."); text = wiki.LoadText(pageName); using (FileStream fs = new FileStream(fileName, FileMode.Create)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress)) using (StreamWriter sw = new StreamWriter(gs)) { sw.WriteLine(page.Attributes["lastrevid"].Value); sw.Write(text); } } day.Page = WikiPage.Parse(pageName, text); List<string> titlesWithResults = new List<string>(); foreach (WikiPageSection section in day.Page.Sections) { RemoveStrikeOut(section); StrikeOutSection(section); Match m = wikiLinkRE.Match(section.Title); if (m.Success && !m.Groups[1].Value.StartsWith(":Категория:")) { string link = m.Groups[1].Value; string movedTo; string movedBy; DateTime movedAt; DateTime start = day.Date; m = timeRE.Match(section.SectionText); if (m.Success) { start = DateTime.Parse(m.Groups[1].Value, CultureInfo.CreateSpecificCulture("ru-RU"), DateTimeStyles.AssumeUniversal); } bool moved = MovedTo(wiki, link, start, out movedTo, out movedBy, out movedAt); WikiPageSection autoVerdictSection = section.Subsections.FirstOrDefault(s => s.Title.Trim() == "Автоматический итог"); if (autoVerdictSection != null && !moved) { autoVerdictSection.Title = " Оспоренный итог "; RemoveStrikeOut(section); StrikeOutSection(section); } if (section.Subsections.Count(s => s.Title.Trim() == "Оспоренный итог") == 0 && section.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") == 0 && section.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") == 0 && moved && !string.IsNullOrEmpty(movedTo)) { string message = string.Format("Страница была переименована {2} в «[[{0}]]» участником [[User:{1}|]]. Данное сообщение было автоматически сгенерировано ботом ~~~~.\n", movedTo, movedBy, movedAt.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)")); WikiPageSection verdict = new WikiPageSection(" Автоматический итог ", section.Level + 1, message); section.AddSubsection(verdict); StrikeOutSection(section); ++results; } } m = wikiLinkRE.Match(section.Title); if (m.Success && section.Title.Contains("<s>")) { titlesWithResults.Add(m.Groups[1].Value); } List<WikiPageSection> sections = new List<WikiPageSection>(); section.Reduce(sections, SubsectionsList); foreach (WikiPageSection subsection in sections) { m = wikiLinkRE.Match(subsection.Title); if (m.Success && subsection.Title.Contains("<s>")) { titlesWithResults.Add(m.Groups[1].Value.Trim()); } if (m.Success && !subsection.Title.Contains("<s>") && !m.Groups[1].Value.StartsWith(":Категория:")) { string link = m.Groups[1].Value; string movedTo; string movedBy; DateTime movedAt; DateTime start = day.Date; m = timeRE.Match(subsection.SectionText); if (m.Success) { start = DateTime.Parse(m.Groups[1].Value, CultureInfo.CreateSpecificCulture("ru-RU"), DateTimeStyles.AssumeUniversal); } bool moved = MovedTo(wiki, link, start, out movedTo, out movedBy, out movedAt); WikiPageSection verdictSection = subsection.Subsections.FirstOrDefault(s => s.Title.Trim() == "Автоматический итог"); if (verdictSection != null && !moved) { verdictSection.Title = " Оспоренный итог "; RemoveStrikeOut(subsection); StrikeOutSection(subsection); } if (subsection.Subsections.Count(s => s.Title.Trim() == "Оспоренный итог") == 0 && subsection.Subsections.Count(s => s.Title.ToLower().Trim() == "итог") == 0 && subsection.Subsections.Count(s => s.Title.Trim() == "Автоматический итог") == 0 && moved && !string.IsNullOrEmpty(movedTo)) { string message = string.Format("Страница была переименована {2} в «[[{0}]]» участником [[User:{1}|]]. Данное сообщение было автоматически сгенерировано ботом ~~~~.\n", movedTo, movedBy, movedAt.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)")); WikiPageSection verdict = new WikiPageSection(" Автоматический итог ", subsection.Level + 1, message); subsection.AddSubsection(verdict); StrikeOutSection(subsection); ++results; } } } } Match matchClosed = closedRE.Match(text); if (matchClosed.Success) { List<TalkResult> talkResults = new List<TalkResult>(); foreach (string name in titlesWithResults) { if (wiki.PageNamespace(name) > 0) { continue; } string movedTo; string movedBy; DateTime movedAt; bool moved = MovedTo(wiki, name, day.Date, out movedTo, out movedBy, out movedAt); talkResults.Add(new TalkResult(name, movedTo, moved)); } parameters.Clear(); parameters.Add("prop", "info"); XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, talkResults.ConvertAll(r => r.Moved ? r.MovedTo : r.Title)); List<string> notificationList = new List<string>(); foreach (XmlNode node in xml.SelectNodes("//page")) { if (node.Attributes["missing"] == null && node.Attributes["ns"].Value == "0") { notificationList.Add(node.Attributes["title"].Value); } } if (notificationList.Count > 0) { parameters.Clear(); parameters.Add("list", "backlinks"); parameters.Add("bltitle", pageName); parameters.Add("blfilterredir", "nonredirects"); parameters.Add("blnamespace", "1"); parameters.Add("bllimit", "max"); XmlDocument backlinks = wiki.Enumerate(parameters, true); foreach (string title in notificationList) { if (backlinks.SelectSingleNode("//bl[@title=\"Обсуждение:" + title + "\"]") == null) { TalkResult tr = talkResults.Find(r => r.Moved ? r.MovedTo == title : r.Title == title); PutNotification(wiki, tr, day.Date); } } } } string newText = day.Page.Text; if (newText.Trim() == text.Trim()) { continue; } try { Console.Out.WriteLine("Updating " + pageName + "..."); string revid = wiki.Save(pageName, "", newText, "зачёркивание заголовков" + (results > 0 ? ", сообщение об итогах" : ""), MinorFlags.Minor, CreateFlags.NoCreate, WatchFlags.None, SaveFlags.Replace, true, basetimestamp, "", editToken); using (FileStream fs = new FileStream(fileName, FileMode.Create)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress)) using (StreamWriter sw = new StreamWriter(gs)) { sw.WriteLine(revid); sw.Write(newText); } } catch (WikiException) { } } }
private void ReplaceEmptyResults(WikiPageSection section) { WikiPageSection result = section.Subsections.FirstOrDefault(s => _l10i.Processor != null ? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim()) : _l10i.Results.Any(r => r == s.Title.Trim())); if (result != null && result.Subsections.Count == 0 && string.IsNullOrEmpty(result.SectionText.Trim())) { result.Title = _l10i.EmptyResult; } section.ForEach(ReplaceEmptyResults); }
private void StrikeOutSection(WikiPageSection section) { Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}"); if (section.Subsections.Count(s => _l10i.Processor != null ? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim()) : _l10i.Results.Any(r => r == s.Title.Trim())) > 0) { if (!section.Title.Contains("<s>")) { section.Title = string.Format(" <s>{0}</s> ", section.Title.Trim()); } foreach (WikiPageSection subsection in section.Subsections) { Match m = wikiLinkRE.Match(subsection.Title); if (m.Success && !subsection.Title.Contains("<s>")) { subsection.Title = string.Format(" <s>{0}</s> ", subsection.Title.Trim()); } } } section.ForEach(StrikeOutSection); }
private void RemoveStrikeOut(WikiPageSection section) { if (section.Subsections.Count(s => s.Title.Trim() == "Итог") == 0) { if (section.Title.Contains("<s>")) { section.Title = section.Title.Replace("<s>", ""); section.Title = section.Title.Replace("</s>", ""); } } section.ForEach(RemoveStrikeOut); }
static string RemoveVotes(WikiPageSection section) { Regex re = new Regex(@"\s+\d{1,3}—\d{1,3}(—\d{1,3})?\s*(</s>)?\s*$"); return re.Replace(section.Title, "$2"); }
public void Parse(string text) { if (_sections.Count != 0) { _sections.Clear(); } if (!string.IsNullOrEmpty(_text)) { _text = ""; } StringReader reader = new StringReader(text); StringBuilder sectionText = new StringBuilder(); int level = 0; string sectionTitle = ""; string rawSectionTitle = ""; bool found = false; Tokens token = Tokens.WikiText; string line; while ((line = reader.ReadLine()) != null) { for (int i = 0; i < line.Length - 1; ++i) { char ch = line[i]; char nextCh = line[i + 1]; switch (token) { case Tokens.WikiText: if (ch == '<' && nextCh == '!') { token = Tokens.HtmlCommentStart; ++i; } else if (ch == '<' && nextCh == 'n') { token = Tokens.NoWikiStartPart1; ++i; } break; case Tokens.HtmlCommentStart: if (ch == '-' && nextCh == '-') { token = Tokens.HtmlComment; ++i; } else { token = Tokens.WikiText; } break; case Tokens.HtmlComment: if (ch == '-' && nextCh == '-') { token = Tokens.HtmlCommentEnd; } break; case Tokens.HtmlCommentEnd: if (ch == '-' && nextCh == '>') { token = Tokens.WikiText; } else if (ch == '-' && nextCh == '-') { break; } else { token = Tokens.HtmlComment; } break; case Tokens.NoWikiStartPart1: if (ch == 'o' && nextCh == 'w') { token = Tokens.NoWikiStartPart2; ++i; } else { token = Tokens.WikiText; } break; case Tokens.NoWikiStartPart2: if (ch == 'i' && nextCh == 'k') { token = Tokens.NoWikiStartPart3; ++i; } else { token = Tokens.WikiText; } break; case Tokens.NoWikiStartPart3: if (ch == 'i' && nextCh == '>') { token = Tokens.NoWiki; ++i; } else { token = Tokens.WikiText; } break; case Tokens.NoWiki: if (ch == '<' && nextCh == '/') { token = Tokens.NoWikiEndPart1; ++i; } break; case Tokens.NoWikiEndPart1: if (ch == 'n' && nextCh == 'o') { token = Tokens.NoWikiEndPart2; ++i; } else { token = Tokens.NoWiki; } break; case Tokens.NoWikiEndPart2: if (ch == 'w' && nextCh == 'i') { token = Tokens.NoWikiEndPart3; ++i; } else { token = Tokens.NoWiki; } break; case Tokens.NoWikiEndPart3: if (ch == 'k' && nextCh == 'i') { token = Tokens.NoWikiEndPart4; } else { token = Tokens.NoWiki; } break; case Tokens.NoWikiEndPart4: if (ch == 'i' && nextCh == '>') { token = Tokens.WikiText; } else { token = Tokens.NoWiki; } break; default: break; } } if (token == Tokens.HtmlCommentStart) { token = Tokens.WikiText; } else if (token == Tokens.HtmlCommentEnd) { token = Tokens.HtmlComment; } else if (token == Tokens.NoWikiStartPart1 || token == Tokens.NoWikiStartPart2 || token == Tokens.NoWikiStartPart3) { token = Tokens.WikiText; } else if (token == Tokens.NoWikiEndPart1 || token == Tokens.NoWikiEndPart2 || token == Tokens.NoWikiEndPart3 || token == Tokens.NoWikiEndPart4) { token = Tokens.NoWiki; } Match m = _sectionRE.Match(line); if (token == Tokens.WikiText && m.Success) { if (found) { var section = new WikiPageSection(sectionTitle, rawSectionTitle, level, sectionText.ToString()); int index = _sections.Count - 1; if (index >= 0 && index < _sections.Count && _sections[index].Level < level) { _sections[index].AddSubsection(section); } else { _sections.Add(section); } sectionText = new StringBuilder(); found = false; } else { _text = sectionText.ToString(); sectionText = new StringBuilder(); } found = true; level = Math.Min(m.Groups[1].Length, m.Groups[3].Length); sectionTitle = m.Groups[2].Value; rawSectionTitle = line; } else { sectionText.Append(line + "\n"); } } if (found) { var section = new WikiPageSection(sectionTitle, rawSectionTitle, level, sectionText.ToString()); int index = _sections.Count - 1; if (index >= 0 && index < _sections.Count && _sections[index].Level < level) { _sections[index].AddSubsection(section); } else { _sections.Add(section); } } else { _text = sectionText.ToString(); } }
public void Run(Wiki wiki) { Directory.CreateDirectory(_cacheDir); Console.Out.WriteLine("Updating " + _page); Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}"); Regex timeRE = new Regex(@"(\d{2}:\d{2}\, \d\d? [а-я]+ \d{4}) \(UTC\)"); ParameterCollection parameters = new ParameterCollection(); parameters.Add("prop", "info|revisions"); parameters.Add("intoken", "edit"); XmlDocument doc = wiki.Query(QueryBy.Titles, parameters, _page); XmlNode page = doc.SelectSingleNode("//page"); string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"); string pageName = page.Attributes["title"].Value; string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value; string editToken = page.Attributes["edittoken"].Value; string starttimestamp = queryTimestamp; string text = ""; string fileName = _cacheDir + WikiCache.EscapePath(_page); if (File.Exists(fileName)) { using (FileStream fs = new FileStream(fileName, FileMode.Open)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress)) using (TextReader sr = new StreamReader(gs)) { string revid = sr.ReadLine(); if (revid == page.Attributes["lastrevid"].Value) { Console.Out.WriteLine("Loading " + pageName + "..."); text = sr.ReadToEnd(); } } } if (string.IsNullOrEmpty(text)) { Console.Out.WriteLine("Downloading " + pageName + "..."); text = wiki.LoadText(pageName); starttimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"); using (FileStream fs = new FileStream(fileName, FileMode.Create)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress)) using (StreamWriter sw = new StreamWriter(gs)) { sw.WriteLine(page.Attributes["lastrevid"].Value); sw.Write(text); } } Dictionary<string, List<WikiPageSection>> titles = new Dictionary<string, List<WikiPageSection>>(); WikiPage wikiPage = WikiPage.Parse(pageName, text); foreach (WikiPageSection section in wikiPage.Sections) { if (section.Subsections.Count(s => s.Title.ToLower(CultureInfo.CreateSpecificCulture("ru-RU")).Trim() == "итог") == 0) { Match m = wikiLinkRE.Match(section.Title); if (m.Success) { string title = m.Groups[1].Value.Trim(); if (titles.ContainsKey(title)) { titles[title].Add(section); } else { titles.Add(title, new List<WikiPageSection>()); titles[title].Add(section); } } } } parameters.Clear(); parameters.Add("prop", "info"); Dictionary<string, string> normalizedTitles = new Dictionary<string, string>(); XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles.Keys); foreach (XmlNode node in xml.SelectNodes("//n")) { normalizedTitles.Add(node.Attributes["to"].Value, node.Attributes["from"].Value); } List<string> notificationList = new List<string>(); XmlNodeList missingTitles = xml.SelectNodes("//page"); foreach (XmlNode node in missingTitles) { string title = node.Attributes["title"].Value; IEnumerable<WikiPageSection> sections; if (titles.ContainsKey(title)) { sections = titles[title]; } else { sections = titles[normalizedTitles[title]]; } if (node.Attributes["missing"] != null) { parameters.Clear(); parameters.Add("list", "logevents"); parameters.Add("letype", "delete"); parameters.Add("lemlimit", "max"); parameters.Add("ledir", "newer"); parameters.Add("letitle", title); parameters.Add("leprop", "comment|type|user|timestamp"); XmlDocument log = wiki.Enumerate(parameters, true); XmlNodeList items = log.SelectNodes("//item"); List<DeleteLogEvent> events = new List<DeleteLogEvent>(); foreach (XmlNode item in items) { DeleteLogEvent ev = new DeleteLogEvent(); ev.Comment = item.Attributes["comment"].Value; ev.Deleted = item.Attributes["action"].Value == "delete"; ev.User = item.Attributes["user"].Value; ev.Timestamp = DateTime.Parse(item.Attributes["timestamp"].Value, null, DateTimeStyles.AssumeUniversal); events.Add(ev); } events.Sort(CompareDeleteLogEvents); if (events.Count > 0 && events[0].Deleted) { string comment = FilterWikiMarkup(events[0].Comment); string message = string.Format("Страница была удалена {1} администратором [[User:{0}|]]. Была указана следующая причина: «{2}». Данное сообщение было автоматически сгенерировано ботом ~~~~.\n", events[0].User, events[0].Timestamp.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)", CultureInfo.CreateSpecificCulture("ru-RU")), comment); var pageSections = titles.ContainsKey(title) ? titles[title] : titles[normalizedTitles[title]]; foreach (WikiPageSection section in pageSections) { WikiPageSection verdict = new WikiPageSection(" Итог ", section.Level + 1, message); section.AddSubsection(verdict); } } } } string newText = wikiPage.Text; if (newText.Trim() == text.Trim()) { return; } Console.Out.WriteLine("Updating " + pageName + "..."); string id = wiki.Save(pageName, "", newText, "автоматическое подведение итогов", MinorFlags.Minor, CreateFlags.NoCreate, WatchFlags.None, SaveFlags.Replace, true, basetimestamp, "", editToken); using (FileStream fs = new FileStream(fileName, FileMode.Create)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress)) using (StreamWriter sw = new StreamWriter(gs)) { sw.WriteLine(id); sw.Write(newText); } }
internal void UpdatePages(Wiki wiki) { ParameterCollection parameters = new ParameterCollection(); parameters.Add("generator", "categorymembers"); parameters.Add("gcmtitle", "Категория:Википедия:Незакрытые обсуждения восстановления страниц"); parameters.Add("gcmlimit", "max"); parameters.Add("gcmnamespace", "4"); parameters.Add("prop", "info|revisions"); parameters.Add("intoken", "edit"); XmlDocument doc = wiki.Enumerate(parameters, true); string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"); XmlNodeList pages = doc.SelectNodes("//page"); foreach (XmlNode page in pages) { string starttimestamp = queryTimestamp; int results = 0; string prefix = "Википедия:К восстановлению/"; string pageName = page.Attributes["title"].Value; string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value; string editToken = page.Attributes["edittoken"].Value; string date = pageName.Substring(prefix.Length); Day day = new Day(); if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture("ru-RU"), DateTimeStyles.AssumeUniversal, out day.Date)) { continue; } string text = ""; string fileName = _cacheDir + date + ".bin"; if (File.Exists(fileName)) { using (FileStream fs = new FileStream(fileName, FileMode.Open)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress)) using (TextReader sr = new StreamReader(gs)) { string revid = sr.ReadLine(); if (revid == page.Attributes["lastrevid"].Value) { Console.Out.WriteLine("Loading " + pageName + "..."); text = sr.ReadToEnd(); } } } if (string.IsNullOrEmpty(text)) { try { Console.Out.WriteLine("Downloading " + pageName + "..."); text = wiki.LoadText(pageName); starttimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"); } catch (WikiPageNotFound) { continue; } using (FileStream fs = new FileStream(fileName, FileMode.Create)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress)) using (StreamWriter sw = new StreamWriter(gs)) { sw.WriteLine(page.Attributes["lastrevid"].Value); sw.Write(text); } } Dictionary<string, List<WikiPageSection>> titles = new Dictionary<string, List<WikiPageSection>>(); day.Page = WikiPage.Parse(pageName, text); foreach (WikiPageSection section in day.Page.Sections) { RemoveStrikeOut(section); StrikeOutSection(section); if (section.Subsections.Count(s => _results.Any(r => r.ToLower() == s.Title.Trim().ToLower())) == 0) { Match m = _wikiLinkRE.Match(section.Title); if (m.Success) { string title = m.Groups[1].Value.Trim(); if (titles.ContainsKey(title)) { titles[title].Add(section); } else { titles.Add(title, new List<WikiPageSection>()); titles[title].Add(section); } } } { List<WikiPageSection> sections = new List<WikiPageSection>(); section.Reduce(sections, SubsectionsList); foreach (WikiPageSection subsection in sections) { Match m = _wikiLinkRE.Match(subsection.Title); if (m.Success && !subsection.Title.Contains("<s>") && subsection.Subsections.Count(s => _results.Any(r => r.ToLower() == s.Title.Trim().ToLower())) == 0) { string title = m.Groups[1].Value.Trim(); if (titles.ContainsKey(title)) { titles[title].Add(subsection); } else { titles.Add(title, new List<WikiPageSection>()); titles[title].Add(subsection); } } } } } parameters.Clear(); parameters.Add("prop", "info"); Dictionary<string, string> normalizedTitles = new Dictionary<string, string>(); XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles.Keys); foreach (XmlNode node in xml.SelectNodes("//n")) { normalizedTitles.Add(node.Attributes["to"].Value, node.Attributes["from"].Value); } List<string> notificationList = new List<string>(); XmlNodeList missingTitles = xml.SelectNodes("//page"); foreach (XmlNode node in missingTitles) { string title = node.Attributes["title"].Value; IEnumerable<WikiPageSection> sections; if (titles.ContainsKey(title)) { sections = titles[title]; } else { sections = titles[normalizedTitles[title]]; } if (node.Attributes["missing"] == null) { DateTime start = day.Date; parameters.Clear(); parameters.Add("list", "logevents"); parameters.Add("letype", "delete"); parameters.Add("lemlimit", "max"); parameters.Add("lestart", start.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")); parameters.Add("ledir", "newer"); parameters.Add("letitle", title); parameters.Add("leprop", "comment|type|user|timestamp"); XmlDocument log = wiki.Enumerate(parameters, true); XmlNodeList items = log.SelectNodes("//item"); List<DeleteLogEvent> events = new List<DeleteLogEvent>(); foreach (XmlNode item in items) { DeleteLogEvent ev = new DeleteLogEvent(); ev.Comment = item.Attributes["comment"].Value; ev.Restored = item.Attributes["action"].Value == "restore"; ev.Deleted = item.Attributes["action"].Value == "delete"; ev.User = item.Attributes["user"].Value; ev.Timestamp = DateTime.Parse(item.Attributes["timestamp"].Value, null, DateTimeStyles.AssumeUniversal); events.Add(ev); } events.Sort(CompareDeleteLogEvents); if (events.Count > 0 && events[0].Restored && (DateTime.Now - events[0].Timestamp).TotalHours > 2) { Regex commentRE = new Regex(@" восстановлено: (.+)"); Match m = commentRE.Match(events[0].Comment); string comment; if (m.Success) { comment = m.Groups[1].Value; } else { comment = "<nowiki>" + events[0].Comment + "</nowiki>"; } string message = string.Format("Страница была восстановлена {1} администратором [[User:{0}|]]. Была указана следующая причина: «{2}». Данное сообщение было автоматически сгенерировано ботом ~~~~.\n", events[0].User, events[0].Timestamp.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)"), comment); if (!titles.ContainsKey(title)) { continue; } foreach (WikiPageSection section in titles[title]) { WikiPageSection verdict = new WikiPageSection(" Автоматический итог ", section.Level + 1, message); section.AddSubsection(verdict); StrikeOutSection(section); ++results; } } else if (events.Count > 0 && events[0].Deleted) { parameters.Clear(); parameters.Add("prop", "revisions"); parameters.Add("rvprop", "timestamp|user"); parameters.Add("rvdir", "newer"); parameters.Add("rvlimit", "1"); parameters.Add("redirects"); log = wiki.Query(QueryBy.Titles, parameters, new string[] { title }); XmlNode revision = log.SelectSingleNode("//rev"); if (revision != null) { string user = revision.Attributes["user"].Value; string timestamp = revision.Attributes["timestamp"].Value; DateTime time = DateTime.Parse(timestamp, null, DateTimeStyles.AssumeUniversal); string message = string.Format("Страница была создана заново {1} участником [[User:{0}|]]. Данное сообщение было автоматически сгенерировано ботом ~~~~.\n", user, time.ToUniversalTime().ToString("d MMMM yyyy в HH:mm (UTC)")); if (!titles.ContainsKey(title)) { continue; } foreach (WikiPageSection section in titles[title]) { WikiPageSection verdict = new WikiPageSection(" Автоматический итог ", section.Level + 1, message); section.AddSubsection(verdict); StrikeOutSection(section); ++results; } } } } } string newText = day.Page.Text; if (newText.Trim() == text.Trim()) { continue; } try { Console.Out.WriteLine("Updating " + pageName + "..."); string revid = wiki.Save(pageName, "", newText, "зачёркивание заголовков" + (results > 0 ? " и подведение итогов" : ""), MinorFlags.Minor, CreateFlags.NoCreate, WatchFlags.None, SaveFlags.Replace, true, basetimestamp, "", editToken); using (FileStream fs = new FileStream(fileName, FileMode.Create)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress)) using (StreamWriter sw = new StreamWriter(gs)) { sw.WriteLine(revid); sw.Write(newText); } } catch (WikiException) { } } }
private void RemoveStrikeOut(WikiPageSection section) { if (section.Subsections.Count(s => _results.Any(r => r.ToLower() == s.Title.Trim().ToLower())) == 0) { if (section.Title.Contains("<s>")) { section.Title = section.Title.Replace("<s>", ""); section.Title = section.Title.Replace("</s>", ""); } } section.ForEach(RemoveStrikeOut); }
protected int SectionsUp(WikiPageSection x, WikiPageSection y) { MatchCollection ms = timeRE.Matches(x.Text); DateTime publishedX = DateTime.MaxValue; foreach (Match match in ms) { string value = match.Groups[1].Value; DateTime time; if (DateTime.TryParse(value, L10i.Culture, DateTimeStyles.AssumeUniversal, out time) && time < publishedX) { publishedX = time; } } if (publishedX == DateTime.MaxValue) { publishedX = DateTime.MinValue; } ms = timeRE.Matches(y.Text); DateTime publishedY = DateTime.MaxValue; foreach (Match match in ms) { string value = match.Groups[1].Value; DateTime time; if (DateTime.TryParse(value, L10i.Culture, DateTimeStyles.AssumeUniversal, out time) && time < publishedY) { publishedY = time; } } if (publishedY == DateTime.MaxValue) { publishedY = DateTime.MinValue; } return publishedY.CompareTo(publishedX); }
protected static bool IsMovedSection(WikiPageSection section) { Regex re = new Regex(@"^\s*\{\{(moved|перенесено на|обсуждение перенесено|moved to|перенесено в)\|(.+?)\}\}\s*$", RegexOptions.IgnoreCase); Match m = re.Match(section.SectionText); return m.Success; }
private void RemoveStrikeOut(WikiPageSection section) { if (!section.Title.ToLower(_l10i.Culture).Contains("{{ok}}") && !section.Title.ToLower(_l10i.Culture).Contains("{{ок}}") && !section.Title.ToLower(_l10i.Culture).Contains("{{x}}") && section.Subsections.Count(s => s.Title.Trim() == "Итог") == 0) { if (section.Title.Contains("<s>")) { section.Title = section.Title.Replace("<s>", ""); section.Title = section.Title.Replace("</s>", ""); } } section.ForEach(RemoveStrikeOut); }
private void StrikeOutSection(WikiPageSection section) { Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}"); if (section.Subsections.Count(s => s.Title.ToLower().Trim() == "итог" || s.Title.Trim() == "Автоматический итог") > 0) { if (!section.Title.Contains("<s>")) { section.Title = string.Format(" <s>{0}</s> ", section.Title.Trim()); } foreach (WikiPageSection subsection in section.Subsections) { Match m = wikiLinkRE.Match(subsection.Title); if (m.Success && !subsection.Title.Contains("<s>")) { subsection.Title = string.Format(" <s>{0}</s> ", subsection.Title.Trim()); } } } section.ForEach(StrikeOutSection); }
public void UpdatePages(Wiki wiki) { Console.Out.WriteLine("Updating articles for deletion..."); Regex wikiLinkRE = new Regex(@"\[{2}(.+?)(\|.+?)?]{2}"); Regex timeRE = new Regex(@"(\d{2}:\d{2}\, \d\d? [а-я]+ \d{4}) \(UTC\)"); ParameterCollection parameters = new ParameterCollection(); parameters.Add("generator", "categorymembers"); parameters.Add("gcmtitle", _l10i.Category); parameters.Add("gcmlimit", "max"); parameters.Add("gcmnamespace", "4"); parameters.Add("prop", "info|revisions"); parameters.Add("intoken", "edit"); XmlDocument doc = wiki.Enumerate(parameters, true); string queryTimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"); XmlNodeList pages = doc.SelectNodes("//page"); foreach (XmlNode page in pages) { string starttimestamp = queryTimestamp; int results = 0; string prefix = _l10i.MainPage + "/"; string pageName = page.Attributes["title"].Value; string basetimestamp = page.FirstChild.FirstChild.Attributes["timestamp"].Value; string editToken = page.Attributes["edittoken"].Value; if (pageName.Length < prefix.Length) { continue; } string date = pageName.Substring(prefix.Length); Day day = new Day(); if (!DateTime.TryParse(date, CultureInfo.CreateSpecificCulture(_l10i.Culture), DateTimeStyles.AssumeUniversal, out day.Date)) { continue; } string text = ""; string fileName = _cacheDir + date + ".bin"; if (File.Exists(fileName)) { using (FileStream fs = new FileStream(fileName, FileMode.Open)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Decompress)) using (TextReader sr = new StreamReader(gs)) { string revid = sr.ReadLine(); if (revid == page.Attributes["lastrevid"].Value) { Console.Out.WriteLine("Loading " + pageName + "..."); text = sr.ReadToEnd(); } } } if (string.IsNullOrEmpty(text)) { try { Console.Out.WriteLine("Downloading " + pageName + "..."); text = wiki.LoadText(pageName); starttimestamp = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"); } catch (WikiPageNotFound) { continue; } using (FileStream fs = new FileStream(fileName, FileMode.Create)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress)) using (StreamWriter sw = new StreamWriter(gs)) { sw.WriteLine(page.Attributes["lastrevid"].Value); sw.Write(text); } } List<string> titlesWithResults = new List<string>(); Dictionary<string, List<WikiPageSection>> titles = new Dictionary<string, List<WikiPageSection>>(); day.Page = WikiPage.Parse(pageName, text); foreach (WikiPageSection section in day.Page.Sections) { ReplaceEmptyResults(section); RemoveStrikeOut(section); StrikeOutSection(section); if (section.Subsections.Count(s => _l10i.Processor != null ? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim()) : _l10i.Results.Any(r => r == s.Title.Trim())) == 0 && section.Subsections.Count(s => s.Title.Trim() == _l10i.ChallengedResult) == 0) { Match m = wikiLinkRE.Match(section.Title); if (m.Success) { string title = m.Groups[1].Value.Trim(); if (titles.ContainsKey(title)) { titles[title].Add(section); } else { titles.Add(title, new List<WikiPageSection>()); titles[title].Add(section); } } } { Match m = wikiLinkRE.Match(section.Title); if (m.Success && section.Title.Contains("<s>")) { titlesWithResults.Add(m.Groups[1].Value.Trim()); } List<WikiPageSection> sections = new List<WikiPageSection>(); section.Reduce(sections, SubsectionsList); foreach (WikiPageSection subsection in sections) { m = wikiLinkRE.Match(subsection.Title); if (m.Success && subsection.Title.Contains("<s>")) { titlesWithResults.Add(m.Groups[1].Value.Trim()); } if (m.Success && !subsection.Title.Contains("<s>") && subsection.Subsections.Count(s => s.Title.Trim() == _l10i.ChallengedResult) == 0 && subsection.Subsections.Count(s => _l10i.Processor != null ? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim()) : _l10i.Results.Any(r => r == s.Title.Trim())) == 0) { string title = m.Groups[1].Value.Trim(); if (titles.ContainsKey(title)) { titles[title].Add(subsection); } else { titles.Add(title, new List<WikiPageSection>()); titles[title].Add(subsection); } } } } } parameters.Clear(); parameters.Add("prop", "info"); Dictionary<string, string> normalizedTitles = new Dictionary<string, string>(); XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, titles.Keys); foreach (XmlNode node in xml.SelectNodes("//n")) { normalizedTitles.Add(node.Attributes["to"].Value, node.Attributes["from"].Value); } List<string> notificationList = new List<string>(); XmlNodeList missingTitles = xml.SelectNodes("//page"); foreach (XmlNode node in missingTitles) { string title = node.Attributes["title"].Value; IEnumerable<WikiPageSection> sections; if (titles.ContainsKey(title)) { sections = titles[title]; } else { sections = titles[normalizedTitles[title]]; } if (node.Attributes["missing"] != null) { DateTime start = day.Date; //foreach (WikiPageSection section in sections) //{ // Match m = timeRE.Match(section.Text); // if (m.Success) // { // start = DateTime.Parse(m.Groups[1].Value, // CultureInfo.CreateSpecificCulture(_l10i.Culture), // DateTimeStyles.AssumeUniversal); // } //} parameters.Clear(); parameters.Add("list", "logevents"); parameters.Add("letype", "delete"); parameters.Add("lemlimit", "max"); parameters.Add("lestart", start.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")); parameters.Add("ledir", "newer"); parameters.Add("letitle", title); parameters.Add("leprop", "comment|type|user|timestamp"); XmlDocument log = wiki.Enumerate(parameters, true); XmlNodeList items = log.SelectNodes("//item"); List<DeleteLogEvent> events = new List<DeleteLogEvent>(); foreach (XmlNode item in items) { DeleteLogEvent ev = new DeleteLogEvent(); ev.Comment = item.Attributes["comment"].Value; ev.Deleted = item.Attributes["action"].Value == "delete"; ev.User = item.Attributes["user"].Value; ev.Timestamp = DateTime.Parse(item.Attributes["timestamp"].Value, null, DateTimeStyles.AssumeUniversal); events.Add(ev); } events.Sort(CompareDeleteLogEvents); if (events.Count > 0 && events[0].Deleted && (DateTime.Now - events[0].Timestamp).TotalHours > 2) { string comment = FilterWikiMarkup(events[0].Comment); string message = string.Format(_l10i.AutoResultMessage, events[0].User, events[0].Timestamp.ToUniversalTime().ToString(_l10i.DateFormat, CultureInfo.CreateSpecificCulture(_l10i.Culture)), comment); var pageSections = titles.ContainsKey(title) ? titles[title] : titles[normalizedTitles[title]]; foreach (WikiPageSection section in pageSections) { WikiPageSection verdict = new WikiPageSection(" " + _l10i.AutoResultSection + " ", section.Level + 1, message); section.AddSubsection(verdict); StrikeOutSection(section); ++results; } } } } if (_l10i.Culture != "ru-RU") { parameters.Clear(); parameters.Add("prop", "info"); xml = wiki.Query(QueryBy.Titles, parameters, titlesWithResults); foreach (XmlNode node in xml.SelectNodes("//page")) { if (node.Attributes["missing"] == null && node.Attributes["redirect"] == null && node.Attributes["ns"].Value == "0") { notificationList.Add(node.Attributes["title"].Value); } } if (notificationList.Count > 0) { parameters.Clear(); parameters.Add("list", "backlinks"); parameters.Add("bltitle", pageName); parameters.Add("blfilterredir", "nonredirects"); parameters.Add("blnamespace", "1"); parameters.Add("bllimit", "max"); XmlDocument backlinks = wiki.Enumerate(parameters, true); foreach (string title in notificationList) { string talkPage = wiki.GetNamespace(1) + ":" + title; if (backlinks.SelectSingleNode("//bl[@title=" + GenerateConcatForXPath(talkPage) + "]") == null) { PutNotification(wiki, title, date); } } } } string newText = day.Page.Text; if (newText.Trim() == text.Trim()) { continue; } try { Console.Out.WriteLine("Updating " + pageName + "..."); string revid = wiki.Save(pageName, "", newText, _l10i.StrikeOutComment + (results > 0 ? _l10i.AutoResultComment : ""), MinorFlags.Minor, CreateFlags.NoCreate, WatchFlags.None, SaveFlags.Replace, true, basetimestamp, "", editToken); using (FileStream fs = new FileStream(fileName, FileMode.Create)) using (GZipStream gs = new GZipStream(fs, CompressionMode.Compress)) using (StreamWriter sw = new StreamWriter(gs)) { sw.WriteLine(revid); sw.Write(newText); } } catch (WikiException) { } } }
private static List<WikiPageSection> SubsectionsList(WikiPageSection section, List<WikiPageSection> aggregator) { Match m = _wikiLinkRE.Match(section.Title); if (m.Success) { aggregator.Add(section); } return section.Reduce(aggregator, SubsectionsList); }
private void RemoveStrikeOut(WikiPageSection section) { if (section.Subsections.Count(s => _l10i.Processor != null ? _l10i.Results.Any(r => r == _l10i.Processor(s).Trim()) : _l10i.Results.Any(r => r == s.Title.Trim())) == 0) { if (section.Title.Contains("<s>")) { section.Title = section.Title.Replace("<s>", ""); section.Title = section.Title.Replace("</s>", ""); } } section.ForEach(RemoveStrikeOut); }
private void StrikeOutSection(WikiPageSection section) { if (section.Subsections.Count(s => s.Title.Trim() == "Итог") > 0) { if (!section.Title.Contains("<s>")) { section.Title = string.Format(" <s>{0}</s> ", section.Title.Trim()); } foreach (WikiPageSection subsection in section.Subsections) { Match m = _wikiLinkRE.Match(subsection.Title); if (m.Success && !subsection.Title.Contains("<s>")) { subsection.Title = string.Format(" <s>{0}</s> ", subsection.Title.Trim()); } } } section.ForEach(StrikeOutSection); }
static string RemoveOK(WikiPageSection section) { Regex re = new Regex(@"^\s*(<s>)?\s*{{(ok|OK|Ok|oK|ОК|ок|Ок|оК|x|X)}}\s*(.+?)(</s>)?\s*$"); return re.Replace(section.Title, "$1$3</s>"); }