public override Dictionary<string, string> Process(Wiki wiki, WikiPage page, ref int diffSize, ref int topicsArchived)
{
List<WikiPageSection> archivedSections = new List<WikiPageSection>();
foreach (WikiPageSection section in page.Sections)
{
WikiPageSection result = section.Subsections.FirstOrDefault(ss => ss.Title.Trim().ToLower() == "итог");
bool forceArchivation = LookForLines.Any(s => section.Text.ToLower().Contains(s.ToLower()));
if (!OnHold.Any(s => section.Text.ToLower().Contains(s.ToLower())) &&
((result != null && !string.IsNullOrEmpty(result.SectionText.Trim())) ||
forceArchivation ||
!CheckForResult))
{
MatchCollection ms = timeRE.Matches(FilterQuotes(section.Text));
DateTime published = DateTime.Today;
DateTime lastReply = DateTime.MinValue;
foreach (Match match in ms)
{
string value = match.Groups[1].Value;
DateTime time = DateTime.Parse(value, L10i.Culture,
DateTimeStyles.AssumeUniversal);
if (time < published)
{
published = time;
}
if (time > lastReply)
{
lastReply = time;
}
}
if (lastReply != DateTime.MinValue &&
((forceArchivation && (DateTime.Today - lastReply).TotalHours >= ForcedArchivationDelay) ||
(DateTime.Today - lastReply).TotalHours >= Delay))
{
archivedSections.Add(section);
}
}
if (IsMovedSection(section))
{
section.SectionText = section.SectionText.Trim(new char[] { ' ', '\t', '\n' }) + "\n~~~~\n";
archivedSections.Add(section);
}
}
Dictionary<string, string> archiveTexts = new Dictionary<string, string>();
if (archivedSections.Count == 0)
{
diffSize = 0;
return archiveTexts;
}
var parameters = new ParameterCollection()
{
{ "prop", "info" },
};
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, MainPage);
int ns = int.Parse(xml.SelectSingleNode("//page").Attributes["ns"].Value);
string prefix = wiki.GetNamespace(ns) + ":";
parameters = new ParameterCollection()
{
{ "list", "allpages" },
{ "apprefix", MainPage.Substring(prefix.Length) },
{ "apnamespace", ns.ToString() }
};
xml = wiki.Enumerate(parameters, true);
int maxNumber = 1;
foreach (XmlNode p in xml.SelectNodes("//p"))
{
string title = p.Attributes["title"].Value;
string format = Format.Replace("{0}", "");
if (title.StartsWith(format))
{
int number;
if (int.TryParse(title.Substring(format.Length), out number))
{
if (number > maxNumber)
{
maxNumber = number;
}
}
}
}
int index = 0;
string pageName = string.Format(Format, maxNumber);
parameters.Clear();
parameters.Add("prop", "info");
xml = wiki.Query(QueryBy.Titles, parameters, new string[] { pageName });
XmlNode node = xml.SelectSingleNode("//page");
if (node.Attributes["missing"] == null)
{
string pageFileName = _cacheDir + Cache.GenerateCachePath(pageName);
string text = Cache.LoadPageFromCache(pageFileName,
node.Attributes["lastrevid"].Value, pageName);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
Cache.CachePage(pageName, _cacheDir, node.Attributes["lastrevid"].Value, text);
}
WikiPage archivePage = WikiPage.Parse(pageName, text);
if (archivePage.Sections.Count < Topics)
{
int topics = Topics - archivePage.Sections.Count;
for (int i = 0; i < topics && index < archivedSections.Count; ++i, ++index)
{
WikiPageSection section = archivedSections[index];
section.Title = ProcessSectionTitle(section.Title);
archivePage.Sections.Add(section);
}
if (NewSectionsDown)
{
archivePage.Sections.Sort(SectionsDown);
}
else
{
archivePage.Sections.Sort(SectionsUp);
}
if (!string.IsNullOrEmpty(RemoveFromText))
{
archivePage.Text = archivePage.Text.Replace(RemoveFromText, "");
}
archiveTexts.Add(pageName, archivePage.Text);
}
}
if (index < archivedSections.Count)
{
string text = Header;
pageName = string.Format(Format, maxNumber + 1);
WikiPage archivePage = WikiPage.Parse(pageName, text);
for (; index < archivedSections.Count; ++index)
{
WikiPageSection section = archivedSections[index];
section.Title = ProcessSectionTitle(section.Title);
archivePage.Sections.Add(section);
}
archivePage.Sections.Sort(SectionsUp);
if (!string.IsNullOrEmpty(RemoveFromText))
{
archivePage.Text = archivePage.Text.Replace(RemoveFromText, "");
}
archiveTexts.Add(pageName, archivePage.Text);
}
topicsArchived = 0;
diffSize = 0;
foreach (var section in archivedSections)
{
diffSize += Encoding.UTF8.GetByteCount(section.Text);
++topicsArchived;
page.Sections.Remove(section);
}
return archiveTexts;
}
public static bool TryParse(WikiPage page,
string directory,
PortalModule portal,
out IPortalModule module)
{
module = null;
Dictionary<string, string> options;
if (!TryParseTemplate(page.Text, out options))
{
return false;
}
var categories = new List<string>();
if (options.ContainsKey("категория"))
{
categories.Add(options["категория"]);
}
if (options.ContainsKey("категории"))
{
string[] separators;
if (options["категории"].Contains("\""))
{
separators = new string[] { "\"," };
}
else
{
separators = new string[] { "," };
}
string[] cats = options["категории"].Split(separators,
StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < cats.Length; ++i)
{
string cat = cats[i].Replace("\"", "").Trim();
if (!string.IsNullOrEmpty(cat))
{
categories.Add(cat);
}
}
}
var categoriesToIgnore = new List<string>();
if (options.ContainsKey("игнорировать"))
{
string[] separators;
if (options["игнорировать"].Contains("\""))
{
separators = new string[] { "\"," };
}
else
{
separators = new string[] { "," };
}
string[] cats = options["игнорировать"].Split(separators,
StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < cats.Length; ++i)
{
string cat = cats[i].Replace("\"", "").Trim();
if (!string.IsNullOrEmpty(cat))
{
categoriesToIgnore.Add(cat);
}
}
}
var usersToIgnore = new List<string>();
if (options.ContainsKey("игнорировать авторов"))
{
string[] separators;
if (options["игнорировать авторов"].Contains("\""))
{
separators = new string[] { "\"," };
}
else
{
separators = new string[] { "," };
}
string[] cats = options["игнорировать авторов"].Split(separators,
StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < cats.Length; ++i)
{
string cat = cats[i].Replace("\"", "").Trim();
if (!string.IsNullOrEmpty(cat))
{
usersToIgnore.Add(cat);
}
}
}
string title = "";
if (options.ContainsKey("страница"))
{
title = options["страница"];
}
string archive = "";
if (options.ContainsKey("архив"))
{
archive = options["архив"];
}
string prefix = "";
if (options.ContainsKey("префикс"))
{
prefix = options["префикс"];
}
bool markEdits = true;
if (options.ContainsKey("помечать правки") && options["помечать правки"].ToLower() == "нет")
{
markEdits = false;
}
int ns = 0;
if (options.ContainsKey("пространство имён"))
{
int.TryParse(options["пространство имён"], out ns);
}
string header = "";
if (options.ContainsKey("шапка"))
{
header = options["шапка"].Replace("\\n", "\n");
}
string footer = "";
if (options.ContainsKey("подвал"))
{
footer = options["подвал"].Replace("\\n", "\n");
}
string templates = "";
if (options.ContainsKey("шаблоны"))
{
templates = options["шаблоны"].Replace("\\n", "\n");
}
string format = "* [[%(название)]]";
if (options.ContainsKey("формат элемента"))
{
format = options["формат элемента"].Replace("{", "{{").Replace("}", "}}");
}
int depth = 15;
if (options.ContainsKey("глубина"))
{
int.TryParse(options["глубина"], out depth);
}
int hours = 720;
if (options.ContainsKey("часов"))
{
int.TryParse(options["часов"], out hours);
}
int maxItems = 20;
if (options.ContainsKey("элементов"))
{
int.TryParse(options["элементов"], out maxItems);
}
int normalSize = 40 * 1000;
if (options.ContainsKey("нормальная"))
{
int.TryParse(options["нормальная"], out normalSize);
}
int shortSize = 10 * 1000;
if (options.ContainsKey("небольшая"))
{
int.TryParse(options["небольшая"], out shortSize);
}
string longColor = "#F2FFF2";
if (options.ContainsKey("цвет крупной"))
{
longColor = options["цвет крупной"];
}
string shortColor = "#FFE8E9";
if (options.ContainsKey("цвет небольшой"))
{
archive = options["цвет небольшой"];
}
string normalColor = "#FFFDE8";
if (options.ContainsKey("цвет нормальной"))
{
archive = options["цвет нормальной"];
}
string delimeter = "\n";
if (options.ContainsKey("разделитель"))
{
delimeter = options["разделитель"].Replace("\"", "").Replace("\\n", "\n");
}
if (options.ContainsKey("тип"))
{
string t = options["тип"].ToLower();
if (t == "список новых статей")
{
if (!options.ContainsKey("архив"))
{
module = new NewPages(portal,
categories,
categoriesToIgnore,
usersToIgnore,
title,
ns,
depth,
hours,
maxItems,
format,
delimeter,
header,
footer,
markEdits);
}
else
{
module = new NewPagesWithArchive(portal,
categories,
categoriesToIgnore,
usersToIgnore,
title,
ns,
archive,
depth,
hours,
maxItems,
format,
delimeter,
header,
footer,
markEdits);
}
}
else if (t == "список наблюдения")
{
module = new WatchList(portal,
categories[0],
title,
ns,
format,
depth);
}
else if (t == "отсортированный список статей, которые должны быть во всех проектах")
{
module = new EncyShell(portal,
title,
shortSize,
normalSize,
shortColor,
normalColor,
longColor);
}
else if (t == "список новых статей с изображениями в карточке")
{
module = new NewPagesWithImages(portal,
categories,
categoriesToIgnore,
title,
ns,
depth,
hours,
maxItems,
format,
delimeter,
header,
footer,
markEdits);
}
else if (t == "список новых статей с изображениями")
{
Regex regex = new Regex(@"\[{2}(Image|File|Файл|Изображение):(?'fileName'.+?)\|");
module = new NewPagesWithImages(portal,
categories,
categoriesToIgnore,
title,
ns,
depth,
hours,
maxItems,
format,
delimeter,
header,
footer,
regex,
markEdits);
}
else if (t == "списки новых статей по дням")
{
module = new NewPagesWithWeeks(portal,
categories,
categoriesToIgnore,
title,
ns,
depth,
maxItems,
format,
delimeter,
header,
footer,
markEdits);
}
else if (t == "список страниц с заданными категориями и шаблонами")
{
module = new CategoryTemplateIntersection(portal,
categories,
categoriesToIgnore,
templates,
title,
ns,
depth,
hours,
maxItems,
format,
delimeter,
header,
footer,
markEdits);
}
else if (t == "список страниц с заданными категориями, шаблонами и обсуждением")
{
module = new CategoryIntersectionAndTalkPages(portal,
categories,
categoriesToIgnore,
templates,
prefix,
title,
ns,
depth,
hours,
maxItems,
format,
delimeter,
header,
footer,
markEdits);
}
}
return module != null;
}
public static WikiPage Parse(string title, string text)
{
WikiPage page = new WikiPage(title);
page.Parse(text);
return page;
}
public static bool TryParse(WikiPage page,
string directory,
bool allowSource,
L10i l10i,
out IArchive archive)
{
archive = null;
Dictionary<string, string> values;
if (!TryParseTemplate(page.Text, out values))
{
return false;
}
int days = 14;
if (values.ContainsKey("срок"))
{
int.TryParse(values["срок"], out days);
}
int forcedArchivationDelay = 0;
if (values.ContainsKey("задержка принудительной архивации"))
{
int.TryParse(values["задержка принудительной архивации"], out forcedArchivationDelay);
}
int minimalSize = 3 * 1024;
if (values.ContainsKey("размер правки"))
{
int.TryParse(values["размер правки"], out minimalSize);
}
int archiveSize = 70 * 1024;
if (values.ContainsKey("размер архива"))
{
int.TryParse(values["размер архива"], out archiveSize);
}
bool checkForResult = false;
if (values.ContainsKey("итог"))
{
string value = values["итог"].ToLower();
if (value == "да")
{
checkForResult = true;
}
}
string pageName = page.Title;
if (allowSource && values.ContainsKey("обрабатывать"))
{
pageName = values["обрабатывать"];
}
string removeFromText = "";
if (values.ContainsKey("убирать из архива"))
{
removeFromText = values["убирать из архива"];
}
string accepted = "";
if (values.ContainsKey("решения"))
{
accepted = values["решения"];
}
string rejected = "";
if (values.ContainsKey("отклонённые заявки"))
{
rejected = values["отклонённые заявки"];
}
string format = pageName + "/Архив/%(номер)";
if (values.ContainsKey("формат"))
{
format = pageName + "/" + values["формат"];
}
string header = "{{closed}}\n";
if (values.ContainsKey("заголовок"))
{
header = values["заголовок"];
}
if (values.ContainsKey("страница"))
{
format = pageName + "/" + values["страница"];
}
if (allowSource && values.ContainsKey("абсолютный путь"))
{
format = values["формат"];
}
int topics = 0;
if (values.ContainsKey("тем в архиве"))
{
int.TryParse(values["тем в архиве"], out topics);
}
bool newSectionsDown = true;
if (values.ContainsKey("новые"))
{
if (values["новые"].ToLower() == "сверху")
{
newSectionsDown = false;
}
}
var onHold = new List<string>();
if (values.ContainsKey("пропускать с"))
{
string[] separators;
if (values["пропускать с"].Contains("\""))
{
separators = new string[] { "\"," };
}
else
{
separators = new string[] { "," };
}
string[] cats = values["пропускать с"].Split(separators,
StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < cats.Length; ++i)
{
string cat = cats[i].Replace("\"", "").Trim();
if (!string.IsNullOrEmpty(cat))
{
onHold.Add(cat);
}
}
}
var lookForLine = new List<string>();
if (values.ContainsKey("архивировать с"))
{
string[] separators;
if (values["архивировать с"].Contains("\""))
{
separators = new string[] { "\"," };
}
else
{
separators = new string[] { "," };
}
string[] cats = values["архивировать с"].Split(separators,
StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < cats.Length; ++i)
{
string cat = cats[i].Replace("\"", "").Trim();
if (!string.IsNullOrEmpty(cat))
{
lookForLine.Add(cat);
}
}
}
if (values.ContainsKey("тип"))
{
string t = values["тип"].ToLower();
if (t == "страница")
{
archive = new Archive(l10i,
pageName,
directory,
days,
format,
header,
lookForLine,
onHold,
removeFromText,
checkForResult,
newSectionsDown,
minimalSize,
forcedArchivationDelay);
}
else if (t == "месяц")
{
archive = new ArchiveByMonth(l10i,
pageName,
directory,
days,
format,
header,
lookForLine,
onHold,
removeFromText,
checkForResult,
newSectionsDown,
minimalSize,
forcedArchivationDelay);
}
else if (t == "год")
{
archive = new ArchiveByYear(l10i,
pageName,
directory,
days,
format,
header,
lookForLine,
onHold,
removeFromText,
checkForResult,
newSectionsDown,
minimalSize,
forcedArchivationDelay);
}
else if (t == "полгода")
{
archive = new ArchiveByHalfYear(l10i,
pageName,
directory,
days,
format,
header,
lookForLine,
onHold,
removeFromText,
checkForResult,
newSectionsDown,
minimalSize,
forcedArchivationDelay);
}
else if (t == "квартал")
{
archive = new ArchiveByQuarter(l10i,
pageName,
directory,
days,
format,
header,
lookForLine,
onHold,
removeFromText,
checkForResult,
newSectionsDown,
minimalSize,
forcedArchivationDelay);
}
else if (t == "нумерация" && topics > 0)
{
archive = new ArchiveByTopicNumber(l10i,
pageName,
directory,
days,
format,
header,
lookForLine,
onHold,
removeFromText,
checkForResult,
newSectionsDown,
topics,
minimalSize,
forcedArchivationDelay);
}
else if (allowSource && t == "заявки на арбитраж")
{
archive = new RequestsForArbitrationArchive(pageName,
accepted,
rejected,
days,
directory);
}
}
if (archive != null)
{
Archive a = archive as Archive;
if (a != null &&
values.ContainsKey("убирать ссылки") &&
values["убирать ссылки"].ToLower() == "да")
{
a.Processor = RemoveHttp;
}
return true;
}
return false;
}
static int Main(string[] args)
{
if (args.Length < 3)
{
Console.Out.WriteLine("ArchiveWikiBot <language> <culture> <update comment>");
return 0;
}
if (string.IsNullOrEmpty(Settings.Default.Login) ||
string.IsNullOrEmpty(Settings.Default.Password))
{
Console.Out.WriteLine("Please add login and password to the configuration file.");
return 0;
}
string path = @"Cache\" + args[0] + @"\";
Directory.CreateDirectory(@"Cache\" + args[0]);
Wiki wiki = new Wiki(string.Format("http://{0}.wikipedia.org/w/", args[0]));
wiki.SleepBetweenQueries = 2;
Console.Out.WriteLine("Logging in as " + Settings.Default.Login + " to " + wiki.Uri + "...");
try
{
string cookieFile = @"Cache\" + args[0] + @"\cookie.jar";
WikiCache.Login(wiki, Settings.Default.Login, Settings.Default.Password, cookieFile);
string namespacesFile = @"Cache\" + args[0] + @"\namespaces.dat";
if (!WikiCache.LoadNamespaces(wiki, namespacesFile))
{
wiki.GetNamespaces();
WikiCache.CacheNamespaces(wiki, namespacesFile);
}
}
catch (WikiException e)
{
Console.Out.WriteLine(e.Message);
return 0;
}
Console.Out.WriteLine("Logged in as " + wiki.User + ".");
L10i l10i = new L10i(args[0], args[1], args[2]);
WikiPage listPage = new WikiPage("User:ClaymoreBot/Архивация/Список");
listPage.Load(wiki);
StringReader reader = new StringReader(listPage.Text);
HashSet<string> pages = new HashSet<string>();
Regex pageRE = new Regex(@"^\*\s*(.+)\s*$");
string line;
while ((line = reader.ReadLine()) != null)
{
Match m = pageRE.Match(line);
if (m.Success)
{
pages.Add(m.Groups[1].Value);
}
}
ParameterCollection parameters = new ParameterCollection();
parameters.Add("generator", "embeddedin");
parameters.Add("geititle", "User:ClaymoreBot/Архивация");
parameters.Add("geilimit", "max");
parameters.Add("prop", "info");
parameters.Add("intoken", "edit");
parameters.Add("redirects");
XmlDocument doc = wiki.Enumerate(parameters, true);
foreach (XmlNode node in doc.SelectNodes("//page"))
{
string title = node.Attributes["title"].Value;
WikiPage page = Cache.Load(wiki, title, path);
IArchive archive;
try
{
if (TryParse(page, path, pages.Contains(page.Title), l10i, out archive))
{
archive.Archivate(wiki);
}
}
catch (WikiException)
{
}
catch (Exception)
{
}
}
return 0;
}
public override Dictionary<string, string> Process(Wiki wiki, WikiPage page, ref int diffSize, ref int topics)
{
Dictionary<string, string> results = new Dictionary<string, string>();
Dictionary<DateTime, List<WikiPageSection>> archives = new Dictionary<DateTime, List<WikiPageSection>>();
List<WikiPageSection> archivedSections = new List<WikiPageSection>();
foreach (WikiPageSection section in page.Sections)
{
WikiPageSection result = section.Subsections.FirstOrDefault(ss => ss.Title.Trim().ToLower() == "итог");
bool forceArchivation = LookForLines.Any(s => section.Text.ToLower().Contains(s.ToLower()));
if (!OnHold.Any(s => section.Text.ToLower().Contains(s.ToLower())) &&
((result != null && !string.IsNullOrEmpty(result.SectionText.Trim())) ||
forceArchivation ||
!CheckForResult))
{
MatchCollection ms = timeRE.Matches(FilterQuotes(section.Text));
DateTime published = NormalizeDate(DateTime.Today);
DateTime lastReply = DateTime.MinValue;
foreach (Match match in ms)
{
string value = match.Groups[1].Value;
DateTime time;
if (!DateTime.TryParse(value, L10i.Culture,
DateTimeStyles.AssumeUniversal, out time))
{
continue;
}
if (time < published)
{
published = NormalizeDate(time);
}
if (time > lastReply)
{
lastReply = time;
}
}
if (lastReply != DateTime.MinValue &&
((forceArchivation && (DateTime.Today - lastReply).TotalHours >= ForcedArchivationDelay) ||
(DateTime.Today - lastReply).TotalHours >= Delay))
{
if (archives.ContainsKey(published))
{
archives[published].Add(section);
}
else
{
List<WikiPageSection> sections = new List<WikiPageSection>();
sections.Add(section);
archives.Add(published, sections);
}
archivedSections.Add(section);
}
}
if (IsMovedSection(section))
{
section.SectionText = section.SectionText.Trim(new char[] { ' ', '\t', '\n' }) + "\n~~~~\n";
}
}
if (archivedSections.Count == 0)
{
return results;
}
foreach (DateTime period in archives.Keys)
{
ParameterCollection parameters = new ParameterCollection();
parameters.Add("prop", "info");
string pageName = DateToPageName(period);
string pageFileName = _cacheDir + Cache.GenerateCachePath(period.ToString(Format));
XmlDocument xml = wiki.Query(QueryBy.Titles, parameters, new string[] { pageName });
XmlNode node = xml.SelectSingleNode("//page");
string text;
if (node.Attributes["missing"] == null)
{
text = Cache.LoadPageFromCache(pageFileName,
node.Attributes["lastrevid"].Value, pageName);
if (string.IsNullOrEmpty(text))
{
Console.Out.WriteLine("Downloading " + pageName + "...");
text = wiki.LoadText(pageName);
Cache.CachePage(period.ToString(Format), _cacheDir, node.Attributes["lastrevid"].Value, text);
}
}
else
{
text = Header;
}
WikiPage archivePage = WikiPage.Parse(pageName, text);
foreach (WikiPageSection section in archives[period])
{
section.Title = ProcessSectionTitle(section.Title);
archivePage.Sections.Add(section);
}
if (NewSectionsDown)
{
archivePage.Sections.Sort(SectionsDown);
}
else
{
archivePage.Sections.Sort(SectionsUp);
}
if (!string.IsNullOrEmpty(RemoveFromText))
{
archivePage.Text = archivePage.Text.Replace(RemoveFromText, "");
}
results.Add(pageName, archivePage.Text);
}
topics = 0;
diffSize = 0;
foreach (var section in archivedSections)
{
diffSize += Encoding.UTF8.GetByteCount(section.Text);
++topics;
page.Sections.Remove(section);
}
return results;
}
public virtual void Save(Wiki wiki, WikiPage page, Dictionary<string, string> archives, int topics)
{
StringBuilder linksToArchives = new StringBuilder();
foreach (var archive in archives)
{
linksToArchives.AppendFormat(", [[{0}]]", archive.Key);
}
linksToArchives.Remove(0, 2);
Console.Out.WriteLine("Saving " + MainPage + "...");
string revid = page.Save(wiki, string.Format("{0} ({1}) → {2}",
L10i.UpdateComment,
topics,
linksToArchives));
if (revid != null)
{
Cache.CachePage(MainPage, _cacheDir, revid, page.Text);
}
foreach (var archive in archives)
{
WikiPage a = new WikiPage(archive.Key, archive.Value);
Console.Out.WriteLine("Saving " + a.Title + "...");
for (int i = 0; i < 5; ++i)
{
try
{
a.Save(wiki, string.Format("{0} ← [[{1}]]", L10i.UpdateComment, page.Title));
break;
}
catch (WikiException)
{
}
}
}
}
