public override Dictionary<string, string> Process(Wiki wiki, WikiPage page, ref int diffSize, ref int topicsArchived) { List<WikiPageSection> archivedSections = new List<WikiPageSection>(); foreach (WikiPageSection section in page.Sections) { WikiPageSection result = section.Subsections.FirstOrDefault(ss => ss.Title.Trim().ToLower() == "итог"); bool forceArchivation = LookForLines.Any(s => section.Text.ToLower().Contains(s.ToLower())); if (!OnHold.Any(s => section.Text.ToLower().Contains(s.ToLower())) && ((result != null && !string.IsNullOrEmpty(result.SectionText.Trim())) || forceArchivation || !CheckForResult)) { MatchCollection ms = timeRE.Matches(FilterQuotes(section.Text)); DateTime published = DateTime.Today; DateTime lastReply = DateTime.MinValue; foreach (Match match in ms) { string value = match.Groups[1].Value; DateTime time = DateTime.Parse(value, L10i.Culture, DateTimeStyles.AssumeUniversal); if (time < published) { published = time; } if (time > lastReply) { lastReply = time; } } if (lastReply != DateTime.MinValue && ((forceArchivation && (DateTime.Today - lastReply).TotalHours >= ForcedArchivationDelay) || (DateTime.Today - lastReply).TotalHours >= Delay)) { archivedSections.Add(section); } } if (IsMovedSection(section)) { section.SectionText = section.SectionText.Trim(new char[] { ' ', '\t', '\n' }) + "\n~~~~\n"; archivedSections.Add(section); } } Dictionary<string, string> archiveTexts = new Dictionary<string, string>(); if (archivedSections.Count == 0) { diffSize = 0; return archiveTexts; } var parameters = new ParameterCollection() { { "prop", "info" }, }; XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, MainPage); int ns = int.Parse(xml.SelectSingleNode("//page").Attributes["ns"].Value); string prefix = wiki.GetNamespace(ns) + ":"; parameters = new ParameterCollection() { { "list", "allpages" }, { "apprefix", MainPage.Substring(prefix.Length) }, { "apnamespace", ns.ToString() } }; xml = wiki.Enumerate(parameters, true); int maxNumber = 1; foreach (XmlNode p in xml.SelectNodes("//p")) { string title = p.Attributes["title"].Value; string format = Format.Replace("{0}", ""); if (title.StartsWith(format)) { int number; if (int.TryParse(title.Substring(format.Length), out number)) { if (number > maxNumber) { maxNumber = number; } } } } int index = 0; string pageName = string.Format(Format, maxNumber); parameters.Clear(); parameters.Add("prop", "info"); xml = wiki.Query(QueryBy.Titles, parameters, new string[] { pageName }); XmlNode node = xml.SelectSingleNode("//page"); if (node.Attributes["missing"] == null) { string pageFileName = _cacheDir + Cache.GenerateCachePath(pageName); string text = Cache.LoadPageFromCache(pageFileName, node.Attributes["lastrevid"].Value, pageName); if (string.IsNullOrEmpty(text)) { Console.Out.WriteLine("Downloading " + pageName + "..."); text = wiki.LoadText(pageName); Cache.CachePage(pageName, _cacheDir, node.Attributes["lastrevid"].Value, text); } WikiPage archivePage = WikiPage.Parse(pageName, text); if (archivePage.Sections.Count < Topics) { int topics = Topics - archivePage.Sections.Count; for (int i = 0; i < topics && index < archivedSections.Count; ++i, ++index) { WikiPageSection section = archivedSections[index]; section.Title = ProcessSectionTitle(section.Title); archivePage.Sections.Add(section); } if (NewSectionsDown) { archivePage.Sections.Sort(SectionsDown); } else { archivePage.Sections.Sort(SectionsUp); } if (!string.IsNullOrEmpty(RemoveFromText)) { archivePage.Text = archivePage.Text.Replace(RemoveFromText, ""); } archiveTexts.Add(pageName, archivePage.Text); } } if (index < archivedSections.Count) { string text = Header; pageName = string.Format(Format, maxNumber + 1); WikiPage archivePage = WikiPage.Parse(pageName, text); for (; index < archivedSections.Count; ++index) { WikiPageSection section = archivedSections[index]; section.Title = ProcessSectionTitle(section.Title); archivePage.Sections.Add(section); } archivePage.Sections.Sort(SectionsUp); if (!string.IsNullOrEmpty(RemoveFromText)) { archivePage.Text = archivePage.Text.Replace(RemoveFromText, ""); } archiveTexts.Add(pageName, archivePage.Text); } topicsArchived = 0; diffSize = 0; foreach (var section in archivedSections) { diffSize += Encoding.UTF8.GetByteCount(section.Text); ++topicsArchived; page.Sections.Remove(section); } return archiveTexts; }
public static bool TryParse(WikiPage page, string directory, PortalModule portal, out IPortalModule module) { module = null; Dictionary<string, string> options; if (!TryParseTemplate(page.Text, out options)) { return false; } var categories = new List<string>(); if (options.ContainsKey("категория")) { categories.Add(options["категория"]); } if (options.ContainsKey("категории")) { string[] separators; if (options["категории"].Contains("\"")) { separators = new string[] { "\"," }; } else { separators = new string[] { "," }; } string[] cats = options["категории"].Split(separators, StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < cats.Length; ++i) { string cat = cats[i].Replace("\"", "").Trim(); if (!string.IsNullOrEmpty(cat)) { categories.Add(cat); } } } var categoriesToIgnore = new List<string>(); if (options.ContainsKey("игнорировать")) { string[] separators; if (options["игнорировать"].Contains("\"")) { separators = new string[] { "\"," }; } else { separators = new string[] { "," }; } string[] cats = options["игнорировать"].Split(separators, StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < cats.Length; ++i) { string cat = cats[i].Replace("\"", "").Trim(); if (!string.IsNullOrEmpty(cat)) { categoriesToIgnore.Add(cat); } } } var usersToIgnore = new List<string>(); if (options.ContainsKey("игнорировать авторов")) { string[] separators; if (options["игнорировать авторов"].Contains("\"")) { separators = new string[] { "\"," }; } else { separators = new string[] { "," }; } string[] cats = options["игнорировать авторов"].Split(separators, StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < cats.Length; ++i) { string cat = cats[i].Replace("\"", "").Trim(); if (!string.IsNullOrEmpty(cat)) { usersToIgnore.Add(cat); } } } string title = ""; if (options.ContainsKey("страница")) { title = options["страница"]; } string archive = ""; if (options.ContainsKey("архив")) { archive = options["архив"]; } string prefix = ""; if (options.ContainsKey("префикс")) { prefix = options["префикс"]; } bool markEdits = true; if (options.ContainsKey("помечать правки") && options["помечать правки"].ToLower() == "нет") { markEdits = false; } int ns = 0; if (options.ContainsKey("пространство имён")) { int.TryParse(options["пространство имён"], out ns); } string header = ""; if (options.ContainsKey("шапка")) { header = options["шапка"].Replace("\\n", "\n"); } string footer = ""; if (options.ContainsKey("подвал")) { footer = options["подвал"].Replace("\\n", "\n"); } string templates = ""; if (options.ContainsKey("шаблоны")) { templates = options["шаблоны"].Replace("\\n", "\n"); } string format = "* [[%(название)]]"; if (options.ContainsKey("формат элемента")) { format = options["формат элемента"].Replace("{", "{{").Replace("}", "}}"); } int depth = 15; if (options.ContainsKey("глубина")) { int.TryParse(options["глубина"], out depth); } int hours = 720; if (options.ContainsKey("часов")) { int.TryParse(options["часов"], out hours); } int maxItems = 20; if (options.ContainsKey("элементов")) { int.TryParse(options["элементов"], out maxItems); } int normalSize = 40 * 1000; if (options.ContainsKey("нормальная")) { int.TryParse(options["нормальная"], out normalSize); } int shortSize = 10 * 1000; if (options.ContainsKey("небольшая")) { int.TryParse(options["небольшая"], out shortSize); } string longColor = "#F2FFF2"; if (options.ContainsKey("цвет крупной")) { longColor = options["цвет крупной"]; } string shortColor = "#FFE8E9"; if (options.ContainsKey("цвет небольшой")) { archive = options["цвет небольшой"]; } string normalColor = "#FFFDE8"; if (options.ContainsKey("цвет нормальной")) { archive = options["цвет нормальной"]; } string delimeter = "\n"; if (options.ContainsKey("разделитель")) { delimeter = options["разделитель"].Replace("\"", "").Replace("\\n", "\n"); } if (options.ContainsKey("тип")) { string t = options["тип"].ToLower(); if (t == "список новых статей") { if (!options.ContainsKey("архив")) { module = new NewPages(portal, categories, categoriesToIgnore, usersToIgnore, title, ns, depth, hours, maxItems, format, delimeter, header, footer, markEdits); } else { module = new NewPagesWithArchive(portal, categories, categoriesToIgnore, usersToIgnore, title, ns, archive, depth, hours, maxItems, format, delimeter, header, footer, markEdits); } } else if (t == "список наблюдения") { module = new WatchList(portal, categories[0], title, ns, format, depth); } else if (t == "отсортированный список статей, которые должны быть во всех проектах") { module = new EncyShell(portal, title, shortSize, normalSize, shortColor, normalColor, longColor); } else if (t == "список новых статей с изображениями в карточке") { module = new NewPagesWithImages(portal, categories, categoriesToIgnore, title, ns, depth, hours, maxItems, format, delimeter, header, footer, markEdits); } else if (t == "список новых статей с изображениями") { Regex regex = new Regex(@"\[{2}(Image|File|Файл|Изображение):(?'fileName'.+?)\|"); module = new NewPagesWithImages(portal, categories, categoriesToIgnore, title, ns, depth, hours, maxItems, format, delimeter, header, footer, regex, markEdits); } else if (t == "списки новых статей по дням") { module = new NewPagesWithWeeks(portal, categories, categoriesToIgnore, title, ns, depth, maxItems, format, delimeter, header, footer, markEdits); } else if (t == "список страниц с заданными категориями и шаблонами") { module = new CategoryTemplateIntersection(portal, categories, categoriesToIgnore, templates, title, ns, depth, hours, maxItems, format, delimeter, header, footer, markEdits); } else if (t == "список страниц с заданными категориями, шаблонами и обсуждением") { module = new CategoryIntersectionAndTalkPages(portal, categories, categoriesToIgnore, templates, prefix, title, ns, depth, hours, maxItems, format, delimeter, header, footer, markEdits); } } return module != null; }
public static WikiPage Parse(string title, string text) { WikiPage page = new WikiPage(title); page.Parse(text); return page; }
public static bool TryParse(WikiPage page, string directory, bool allowSource, L10i l10i, out IArchive archive) { archive = null; Dictionary<string, string> values; if (!TryParseTemplate(page.Text, out values)) { return false; } int days = 14; if (values.ContainsKey("срок")) { int.TryParse(values["срок"], out days); } int forcedArchivationDelay = 0; if (values.ContainsKey("задержка принудительной архивации")) { int.TryParse(values["задержка принудительной архивации"], out forcedArchivationDelay); } int minimalSize = 3 * 1024; if (values.ContainsKey("размер правки")) { int.TryParse(values["размер правки"], out minimalSize); } int archiveSize = 70 * 1024; if (values.ContainsKey("размер архива")) { int.TryParse(values["размер архива"], out archiveSize); } bool checkForResult = false; if (values.ContainsKey("итог")) { string value = values["итог"].ToLower(); if (value == "да") { checkForResult = true; } } string pageName = page.Title; if (allowSource && values.ContainsKey("обрабатывать")) { pageName = values["обрабатывать"]; } string removeFromText = ""; if (values.ContainsKey("убирать из архива")) { removeFromText = values["убирать из архива"]; } string accepted = ""; if (values.ContainsKey("решения")) { accepted = values["решения"]; } string rejected = ""; if (values.ContainsKey("отклонённые заявки")) { rejected = values["отклонённые заявки"]; } string format = pageName + "/Архив/%(номер)"; if (values.ContainsKey("формат")) { format = pageName + "/" + values["формат"]; } string header = "{{closed}}\n"; if (values.ContainsKey("заголовок")) { header = values["заголовок"]; } if (values.ContainsKey("страница")) { format = pageName + "/" + values["страница"]; } if (allowSource && values.ContainsKey("абсолютный путь")) { format = values["формат"]; } int topics = 0; if (values.ContainsKey("тем в архиве")) { int.TryParse(values["тем в архиве"], out topics); } bool newSectionsDown = true; if (values.ContainsKey("новые")) { if (values["новые"].ToLower() == "сверху") { newSectionsDown = false; } } var onHold = new List<string>(); if (values.ContainsKey("пропускать с")) { string[] separators; if (values["пропускать с"].Contains("\"")) { separators = new string[] { "\"," }; } else { separators = new string[] { "," }; } string[] cats = values["пропускать с"].Split(separators, StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < cats.Length; ++i) { string cat = cats[i].Replace("\"", "").Trim(); if (!string.IsNullOrEmpty(cat)) { onHold.Add(cat); } } } var lookForLine = new List<string>(); if (values.ContainsKey("архивировать с")) { string[] separators; if (values["архивировать с"].Contains("\"")) { separators = new string[] { "\"," }; } else { separators = new string[] { "," }; } string[] cats = values["архивировать с"].Split(separators, StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < cats.Length; ++i) { string cat = cats[i].Replace("\"", "").Trim(); if (!string.IsNullOrEmpty(cat)) { lookForLine.Add(cat); } } } if (values.ContainsKey("тип")) { string t = values["тип"].ToLower(); if (t == "страница") { archive = new Archive(l10i, pageName, directory, days, format, header, lookForLine, onHold, removeFromText, checkForResult, newSectionsDown, minimalSize, forcedArchivationDelay); } else if (t == "месяц") { archive = new ArchiveByMonth(l10i, pageName, directory, days, format, header, lookForLine, onHold, removeFromText, checkForResult, newSectionsDown, minimalSize, forcedArchivationDelay); } else if (t == "год") { archive = new ArchiveByYear(l10i, pageName, directory, days, format, header, lookForLine, onHold, removeFromText, checkForResult, newSectionsDown, minimalSize, forcedArchivationDelay); } else if (t == "полгода") { archive = new ArchiveByHalfYear(l10i, pageName, directory, days, format, header, lookForLine, onHold, removeFromText, checkForResult, newSectionsDown, minimalSize, forcedArchivationDelay); } else if (t == "квартал") { archive = new ArchiveByQuarter(l10i, pageName, directory, days, format, header, lookForLine, onHold, removeFromText, checkForResult, newSectionsDown, minimalSize, forcedArchivationDelay); } else if (t == "нумерация" && topics > 0) { archive = new ArchiveByTopicNumber(l10i, pageName, directory, days, format, header, lookForLine, onHold, removeFromText, checkForResult, newSectionsDown, topics, minimalSize, forcedArchivationDelay); } else if (allowSource && t == "заявки на арбитраж") { archive = new RequestsForArbitrationArchive(pageName, accepted, rejected, days, directory); } } if (archive != null) { Archive a = archive as Archive; if (a != null && values.ContainsKey("убирать ссылки") && values["убирать ссылки"].ToLower() == "да") { a.Processor = RemoveHttp; } return true; } return false; }
static int Main(string[] args) { if (args.Length < 3) { Console.Out.WriteLine("ArchiveWikiBot <language> <culture> <update comment>"); return 0; } if (string.IsNullOrEmpty(Settings.Default.Login) || string.IsNullOrEmpty(Settings.Default.Password)) { Console.Out.WriteLine("Please add login and password to the configuration file."); return 0; } string path = @"Cache\" + args[0] + @"\"; Directory.CreateDirectory(@"Cache\" + args[0]); Wiki wiki = new Wiki(string.Format("http://{0}.wikipedia.org/w/", args[0])); wiki.SleepBetweenQueries = 2; Console.Out.WriteLine("Logging in as " + Settings.Default.Login + " to " + wiki.Uri + "..."); try { string cookieFile = @"Cache\" + args[0] + @"\cookie.jar"; WikiCache.Login(wiki, Settings.Default.Login, Settings.Default.Password, cookieFile); string namespacesFile = @"Cache\" + args[0] + @"\namespaces.dat"; if (!WikiCache.LoadNamespaces(wiki, namespacesFile)) { wiki.GetNamespaces(); WikiCache.CacheNamespaces(wiki, namespacesFile); } } catch (WikiException e) { Console.Out.WriteLine(e.Message); return 0; } Console.Out.WriteLine("Logged in as " + wiki.User + "."); L10i l10i = new L10i(args[0], args[1], args[2]); WikiPage listPage = new WikiPage("User:ClaymoreBot/Архивация/Список"); listPage.Load(wiki); StringReader reader = new StringReader(listPage.Text); HashSet<string> pages = new HashSet<string>(); Regex pageRE = new Regex(@"^\*\s*(.+)\s*$"); string line; while ((line = reader.ReadLine()) != null) { Match m = pageRE.Match(line); if (m.Success) { pages.Add(m.Groups[1].Value); } } ParameterCollection parameters = new ParameterCollection(); parameters.Add("generator", "embeddedin"); parameters.Add("geititle", "User:ClaymoreBot/Архивация"); parameters.Add("geilimit", "max"); parameters.Add("prop", "info"); parameters.Add("intoken", "edit"); parameters.Add("redirects"); XmlDocument doc = wiki.Enumerate(parameters, true); foreach (XmlNode node in doc.SelectNodes("//page")) { string title = node.Attributes["title"].Value; WikiPage page = Cache.Load(wiki, title, path); IArchive archive; try { if (TryParse(page, path, pages.Contains(page.Title), l10i, out archive)) { archive.Archivate(wiki); } } catch (WikiException) { } catch (Exception) { } } return 0; }
public override Dictionary<string, string> Process(Wiki wiki, WikiPage page, ref int diffSize, ref int topics) { Dictionary<string, string> results = new Dictionary<string, string>(); Dictionary<DateTime, List<WikiPageSection>> archives = new Dictionary<DateTime, List<WikiPageSection>>(); List<WikiPageSection> archivedSections = new List<WikiPageSection>(); foreach (WikiPageSection section in page.Sections) { WikiPageSection result = section.Subsections.FirstOrDefault(ss => ss.Title.Trim().ToLower() == "итог"); bool forceArchivation = LookForLines.Any(s => section.Text.ToLower().Contains(s.ToLower())); if (!OnHold.Any(s => section.Text.ToLower().Contains(s.ToLower())) && ((result != null && !string.IsNullOrEmpty(result.SectionText.Trim())) || forceArchivation || !CheckForResult)) { MatchCollection ms = timeRE.Matches(FilterQuotes(section.Text)); DateTime published = NormalizeDate(DateTime.Today); DateTime lastReply = DateTime.MinValue; foreach (Match match in ms) { string value = match.Groups[1].Value; DateTime time; if (!DateTime.TryParse(value, L10i.Culture, DateTimeStyles.AssumeUniversal, out time)) { continue; } if (time < published) { published = NormalizeDate(time); } if (time > lastReply) { lastReply = time; } } if (lastReply != DateTime.MinValue && ((forceArchivation && (DateTime.Today - lastReply).TotalHours >= ForcedArchivationDelay) || (DateTime.Today - lastReply).TotalHours >= Delay)) { if (archives.ContainsKey(published)) { archives[published].Add(section); } else { List<WikiPageSection> sections = new List<WikiPageSection>(); sections.Add(section); archives.Add(published, sections); } archivedSections.Add(section); } } if (IsMovedSection(section)) { section.SectionText = section.SectionText.Trim(new char[] { ' ', '\t', '\n' }) + "\n~~~~\n"; } } if (archivedSections.Count == 0) { return results; } foreach (DateTime period in archives.Keys) { ParameterCollection parameters = new ParameterCollection(); parameters.Add("prop", "info"); string pageName = DateToPageName(period); string pageFileName = _cacheDir + Cache.GenerateCachePath(period.ToString(Format)); XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, new string[] { pageName }); XmlNode node = xml.SelectSingleNode("//page"); string text; if (node.Attributes["missing"] == null) { text = Cache.LoadPageFromCache(pageFileName, node.Attributes["lastrevid"].Value, pageName); if (string.IsNullOrEmpty(text)) { Console.Out.WriteLine("Downloading " + pageName + "..."); text = wiki.LoadText(pageName); Cache.CachePage(period.ToString(Format), _cacheDir, node.Attributes["lastrevid"].Value, text); } } else { text = Header; } WikiPage archivePage = WikiPage.Parse(pageName, text); foreach (WikiPageSection section in archives[period]) { section.Title = ProcessSectionTitle(section.Title); archivePage.Sections.Add(section); } if (NewSectionsDown) { archivePage.Sections.Sort(SectionsDown); } else { archivePage.Sections.Sort(SectionsUp); } if (!string.IsNullOrEmpty(RemoveFromText)) { archivePage.Text = archivePage.Text.Replace(RemoveFromText, ""); } results.Add(pageName, archivePage.Text); } topics = 0; diffSize = 0; foreach (var section in archivedSections) { diffSize += Encoding.UTF8.GetByteCount(section.Text); ++topics; page.Sections.Remove(section); } return results; }
public virtual void Save(Wiki wiki, WikiPage page, Dictionary<string, string> archives, int topics) { StringBuilder linksToArchives = new StringBuilder(); foreach (var archive in archives) { linksToArchives.AppendFormat(", [[{0}]]", archive.Key); } linksToArchives.Remove(0, 2); Console.Out.WriteLine("Saving " + MainPage + "..."); string revid = page.Save(wiki, string.Format("{0} ({1}) → {2}", L10i.UpdateComment, topics, linksToArchives)); if (revid != null) { Cache.CachePage(MainPage, _cacheDir, revid, page.Text); } foreach (var archive in archives) { WikiPage a = new WikiPage(archive.Key, archive.Value); Console.Out.WriteLine("Saving " + a.Title + "..."); for (int i = 0; i < 5; ++i) { try { a.Save(wiki, string.Format("{0} ← [[{1}]]", L10i.UpdateComment, page.Title)); break; } catch (WikiException) { } } } }