public void Execute(MediaWiki wiki, string[] commandLine)
{
if (commandLine.Length != 1)
{
throw new Exception("specify user name as a single parameter");
}
var rules = _rules[commandLine.Single()];
var talkName = string.Format("Обсуждение участника:{0}", rules.UserName);
var now = DateTimeOffset.UtcNow;
var archiveName = string.Format("{0}/{1}", talkName, rules.GetArchiveName(now.Date));
var talks = new Talks(wiki.GetPage(talkName));
var dayX = now.AddDays(-rules.AgeLimitInDays);
var removed = new Talks();
foreach (var talk in talks.ToArray())
{
if (talk.LastActivity.HasValue && talk.LastActivity < dayX)
{
talks.Remove(talk);
removed.Add(talk);
}
}
if (!removed.Any())
{
return;
}
wiki.Edit(archiveName, "\n\n" + removed.FullText, EditSummary, true);
wiki.Edit(talkName, talks.FullText, $"[[{archiveName}|{EditSummary}]]");
}
private static string GetMailText(MediaWiki wiki)
{
var text = wiki.GetPage(IssuePage, followRedirects: true);
var issues = new SectionedArticle <Section>(text, 2);
var sb = new StringBuilder();
foreach (var issue in issues)
{
var items = GetItems(new SectionedArticle <Section>(issue.Text, 3));
if (items.Length == 0)
{
continue;
}
sb.AppendFormat(@"== '''Сайнпост-дайджест.''' {0} ==
[[Проект:Сайнпост-Дайджест|В этом номере]]:
", StripRefs(GetTitle(issue)));
foreach (var item in items)
{
sb.AppendFormat("* {0}\n", item);
}
}
if (sb.Length == 0)
{
return(null);
}
sb.AppendLine("\n'''<small>Сообщение оставлено [[У:Lê Lợi (bot)|ботом]]. Чтобы отписаться, удалите эту страницу из [[Проект:Сайнпост-Дайджест/Подписка|списка рассылки]].</small>''' ~~~~~");
return(sb.ToString());
}
private static DateTimeOffset?GetValidThroughTime(MediaWiki wiki, string title, DateTimeOffset date)
{
var oldDate = date.AddMonths(-3);
var history = wiki.GetHistory(title, oldDate);
if (history == null)
{
return(null);// article doesn't exist
}
if (history.Length == 0)
{
return(DateTimeOffset.MinValue); // no revisions for 3 months, thus not valid
}
// retreiving immediate parent of the first "3 months old" edit
// or faking the "creation" with a zero-sized revision
var parentId = history.First().ParentId;
var parent = parentId == 0 ? null : wiki.GetRevisionInfo(parentId);
parent = parent ?? new MediaWiki.RevisionInfo();
parent.Timestamp = oldDate.AddTicks(-1);
// searching for the first revision that makes article non-valid
foreach (var entry in new[] { parent }.Concat(history))
{
var newDate = entry.Timestamp.AddMonths(3);
var newSize = GetArticleSize(newDate, history);
if (entry.Size * 2 > newSize)
{
return(newDate == parent.Timestamp.AddMonths(3) ? DateTimeOffset.MinValue : newDate);
}
}
throw new Exception("Impossible...");
}
public void Execute(MediaWiki wiki, string[] commandLine)
{
var retries = 5;
for (var i = 0; i < retries; i++)
{
try
{
var page = wiki.GetPageInfo(DidYouKnow.NextIssueName);
var preparation = new NextIssuePreparation(page.Text);
if (CheckPreparation(preparation, wiki, commandLine.Contains("-onlyNew")))
{
wiki.Edit(DidYouKnow.NextIssueName, preparation.FullText, "Автоматическое обновление страницы.", revId: page.Id);
}
return;
}
catch (MediaWikiApiException ex)
{
if (ex.Code == MediaWiki.ErrorCode.EditConflict)
{
continue;
}
throw;
}
}
throw new Exception($"Edit conflict after {retries} retries");
}
private bool CheckPreparation(NextIssuePreparation preparation, MediaWiki wiki, bool onlyNew)
{
var hasChanges = false;
if (preparation.NewSections != null && preparation.NewSections.Count > 0)
{
preparation.Sections.InsertRange(0, preparation.NewSections);
preparation.NewSections.Clear();
hasChanges = true;
}
foreach (var article in preparation.Sections.SelectMany(s => s.Articles).ToArray())
{
if (onlyNew && article.Status != null)
{
continue;
}
if (article.Status != null && article.Status.Extra != null)
{
continue;
}
article.Status = CheckStatus(wiki, article.Title) ?? CheckValidness(wiki, article.Title);
hasChanges = true;
}
if (hasChanges)
{
preparation.Update();
}
return(hasChanges);
}
public string GetText(MediaWiki wiki)
{
if (_text == null)
{
_text = wiki.GetPage(_info.Id);
}
return(_text);
}
private DYKStatusTemplate CheckValidness(MediaWiki wiki, string title)
{
var validThrough = GetValidThroughTime(wiki, title, DateTimeOffset.UtcNow);
return(validThrough == null
? DYKStatusTemplate.Missing()
: DYKStatusTemplate.Valid(validThrough.Value));
}
public void Execute(MediaWiki wiki, string[] commandLine)
{
var allTemplateNames = wiki.GetAllPageNames("Template:" + TemplateName);
var titles = wiki.GetPagesInCategory(CategoryName, TemplateNamespaceId);
foreach (var title in titles)
{
var history = Revision.FromHistory(wiki.GetHistory(title, DateTimeOffset.MinValue));
LoadUsers(wiki, history);
var page = new ParserUtils(wiki).FindTemplates(history.First().GetText(wiki), allTemplateNames);
foreach (var dt in page.Select(t => new DewikifyTemplate(t)).ToArray())
{
if (dt.Error != null)
{
page.Update(dt.Template, string.Format("<span style='color: red'>Ошибка в шаблоне <nowiki>{0}</nowiki>: '''{1}'''</span>", dt.Template.ToString(), dt.Error));
continue;
}
if (dt.IsDone)
{
continue;
}
var section = GetSectionName(page, dt.Template);
if (section != ClosingSectionName)
{
page.Update(dt.Template, string.Format("<span style='color: red'>Шаблон <nowiki>{0}</nowiki> должен находиться в секции '''Итоги'''.</span>", dt.Template.ToString()));
continue;
}
var user = GetUser(wiki, dt, allTemplateNames, history);
if (!_powerUsers[user])
{
page.Update(dt.Template, string.Format("<span style='color: red'>Шаблон <nowiki>{0}</nowiki> установлен пользователем {{{{u|{1}}}}}, не имеющим флага ПИ/А.</span>", dt.Template.ToString(), user));
continue;
}
var allTitles = wiki.GetAllPageNames(dt.Title);
Dewikify(wiki, allTitles, dt.Title);
foreach (var t in allTitles.Skip(1))
{
wiki.Delete(t, string.Format("[[ВП:КБУ#П1]] [[{0}]]", dt.Title));
}
dt.IsDone = true;
page.Update(dt.Template, dt.Template.ToString());
}
if (history.First().GetText(wiki) != page.Text)
{
wiki.Edit(title, page.Text, Summary);
}
}
}
private void StabilizeArticle(MediaWiki wiki, string page)
{
var text = wiki.GetPage(page);
if (text != null && StatusTemplateRegex.IsMatch(text))
{
wiki.Stabilize(page, "Автоматическая стабилизация хорошей статьи.", null);
}
}
private string GetUser(MediaWiki wiki, DewikifyTemplate template, string[] allTemplateNames, Revision[] history)
{
// looking for the first edit where the template did not exist
return(history.SkipWhile(wiki, text => new ParserUtils(wiki).FindTemplates(text, allTemplateNames)
.Select(t => new DewikifyTemplate(t))
.Where(t => t.Error == null)
.Any(t => t.Title == template.Title)).Info.User);
}
private void LoadUsers(MediaWiki wiki, Revision[] history)
{
var users = wiki.GetUserGroups(history.Select(h => h.Info.User).Distinct().Except(_powerUsers.Keys).ToArray());
foreach (var user in users)
{
var groups = user.Value;
_powerUsers.Add(user.Key, groups.Any(g => IncludeGroups.Contains(g)) && groups.All(g => !ExcludeGroups.Contains(g)));
}
}
public static void LoadText(MediaWiki wiki, IEnumerable <Revision> revisions)
{
revisions = revisions.Where(r => r._text == null).ToArray();
var texts = wiki.GetPages(revisions.Select(r => r.Info.Id).ToArray());
foreach (var rev in revisions)
{
rev._text = texts[rev.Info.Id];
}
}
private IEnumerable <string> GetSubscribers(MediaWiki wiki)
{
var text = wiki.GetPage(SubscribersPage, followRedirects: true);
var section = new SectionedArticle <Section>(text, 2).Single();
foreach (var line in section.Text.Split('\n'))
{
if (!line.StartsWith("*"))
{
continue;
}
yield return(ParserUtils.FindAnyLinks(line).Single());
}
}
private void Stabilize(MediaWiki wiki, string templateTitle, DateTimeOffset?expiry)
{
var links = ParserUtils.FindLinks(wiki.GetPage(templateTitle));
var normalized = wiki.Normalize(links);
foreach (var article in links.Select(x => normalized.TryGetValue(x)).Where(x => x != null).Distinct())
{
DateTimeOffset?e;
if (wiki.GetStabilizationExpiry(article, out e))
{
continue;
}
wiki.Stabilize(article, "Автоматическая стабилизация статьи из актуальных событий", expiry);
}
}
public void Execute(MediaWiki wiki, string[] commandLine)
{
var text = GetMailText(wiki);
if (text == null)
{
return;
}
text = "\n\n" + text;
foreach (var user in GetSubscribers(wiki))
{
wiki.Edit(user, text, Summary, true);
}
}
public void Execute(MediaWiki wiki, string[] commandLine)
{
var text = wiki.GetPage(PageName);
var template = new ParserUtils(wiki).FindTemplates(text, TemplateName).FirstOrDefault();
if (template == null)
{
throw new Exception("Current featured list template not found.");
}
for (var i = 0; i < 2; i++)
{
wiki.Stabilize(template[i].Trim(), "Автоматическая стабилизация избранного списка.", null);
}
}
public MediaWiki ParseMediaWikiFromXML(string xmlFilePath)
{
MediaWiki mediaWiki = new MediaWiki();
XmlDocument doc = new XmlDocument();
doc.Load(xmlFilePath);
mediaWiki.Pages = new List <Page>();
foreach (XmlNode node in doc.DocumentElement)
{
if (node.Name == "page")
{
Page page = new Page();
foreach (XmlNode child in node.ChildNodes)
{
switch (child.Name)
{
case "title":
page.Title = child.InnerText;
break;
case "revision":
Revision revision = new Revision();
foreach (XmlNode revisionNode in child.ChildNodes)
{
switch (revisionNode.Name)
{
case "text":
revision.Text = revisionNode.InnerText;
break;
}
}
page.Revision = revision;
break;
default:
break;
}
}
mediaWiki.Pages.Add(page);
}
}
return(mediaWiki);
}
static void Main(string[] args)
{
WikiExtractor wiki = new WikiExtractor();
MediaWiki mediaWiki = wiki.ParseMediaWikiFromXML(@"C:\Users\Alexander\Downloads\Wikipedia-20181014182236.xml");
Extractor extractor = new Extractor();
List <Denomination> denominations = new List <Denomination>();
foreach (Page page in mediaWiki.Pages)
{
Denomination denomination = extractor.GetDenominationFromPage(page);
denominations.Add(denomination);
}
Console.WriteLine("Done");
}
private static MediaWiki LogIntoWiki()
{
var browser = new Browser {
UserAgent = GetUserAgent(), Cookies = CookieJar.Load()
};
var wiki = new MediaWiki(new Uri("https://ru.wikipedia.org/w/api.php"), browser);
if (!wiki.IsLoggedIn())
{
var creds = ReadCredentials();
wiki.Login(creds.Login, creds.Password);
CookieJar.Save(browser.Cookies);
}
return(wiki);
}
private void TestMediaWikiAccountThread(MediaWikiAccount acc, Action success, Action <string> failure)
{
try
{
MediaWiki connector = new MediaWiki(new MediaWikiOptions(acc, null)); // TODO: MediaWiki CheckProxySettings().GetWebProxy
connector.Login();
success();
}
catch (Exception ex)
{
// ignore ThreadInterruptedException : the request timed out and the thread was interrupted
if (!(ex.InnerException is ThreadInterruptedException))
{
failure(ex.Message);
}
}
}
private static async Task UpdateTemplate(MediaWiki wiki, UserInfo user, string title, string page, JObject settings)
{
var template = new Template {
Name = settings.Value <string>("name")
};
var args = settings.Value <JArray>("args");
foreach (var arg in args.Values <JObject>())
{
foreach (var prop in arg.Properties())
{
prop.Value = prop.Value.Value <string>().Replace("%user.name%", user.Username);
}
template.Args.Add(arg.ToObject <Template.Argument>());
}
var templateString = template + "\n";
if (settings.Value <bool>("talkPage"))
{
title = "Talk:" + title;
page = await wiki.GetPage(title) ?? "";
}
var existing = ParserUtils.FindTemplates(page, template.Name);
if (existing.Count == 0)
{
page = page.Insert(0, templateString);
}
else
{
var regions = existing.Values.OrderByDescending(r => r.Offset).ToArray();
foreach (var r in regions)
{
var region = ParserUtils.ExpandToWholeLine(page, r);
page = page.Remove(region.Offset, region.Length);
}
page = page.Insert(regions.Last().Offset, templateString);
}
await wiki.EditPage(title, page, "Automatically adding template");
}
private void Dewikify(MediaWiki wiki, string[] titles, string originalTitle, IDictionary <string, string[]> entries, Func <string, string, ParserUtils, string> dewikify)
{
var parser = new ParserUtils(wiki);
var linkingPages = entries.Values.SelectMany(x => x).Distinct().ToArray();
foreach (var page in wiki.GetPages(linkingPages).Values.Where(p => p != null))
{
var text = page.Text;
foreach (var title in titles)
{
text = dewikify(text, title, parser);
}
if (text != page.Text)
{
wiki.Edit(page.Title, text, string.Format(SummaryWithTitle, originalTitle));
}
}
}
private static async Task UpdateTemplate(MediaWiki wiki, UserInfo user, string title, string page, TemplateConfig settings, string message)
{
var template = new Template {
Name = settings.Name
};
foreach (var arg in settings.Args)
{
template.Args.Add(new Template.Argument
{
Name = arg.Name,
Value = arg.Value.Replace("%user.name%", user.Username),
});
}
if (settings.TalkPage)
{
title = "Talk:" + title;
page = await wiki.GetPage(title) ?? "";
}
var templateString = template + "\n";
var existing = ParserUtils.FindTemplates(page, template.Name);
if (existing.Count == 0)
{
page = page.Insert(0, templateString);
}
else
{
var regions = existing.Values.OrderByDescending(r => r.Offset).ToArray();
foreach (var r in regions)
{
var region = ParserUtils.ExpandToWholeLine(page, r);
page = page.Remove(region.Offset, region.Length);
}
page = page.Insert(regions.Last().Offset, templateString);
}
await wiki.EditPage(title, page, message);
}
public UploadResult UploadImage(string localFilePath)
{
// Create the connector
MediaWiki connector = new MediaWiki(this.Options);
// Get the file name to save
string filename = Path.GetFileName(localFilePath);
// Upload the image
connector.UploadImage(localFilePath, filename);
string remotePath = Options.Account.Url + "/index.php?title=Image:" + filename;
// Create the file manager object
UploadResult ur = new UploadResult()
{
URL = remotePath
};
return(ur);
}
private DYKStatusTemplate CheckStatus(MediaWiki wiki, string title)
{
var text = wiki.GetPage(title, followRedirects: true);
if (text == null)
{
return(DYKStatusTemplate.Missing());
}
else if (ForDeletionRegex.IsMatch(text))
{
return(DYKStatusTemplate.ForDeletion());
}
else if (Encoding.UTF8.GetByteCount(text) < MinArticleSize)
{
return(DYKStatusTemplate.Small());
}
else
{
return(null);
}
}
public void Execute(MediaWiki wiki, string[] args)
{
var nextIssueDate = GetNearestIssueDate();
var prevIssueDate = nextIssueDate.AddDays(-DYKPeriod);
if (!args.Contains("-force") && (Now() - nextIssueDate).Duration() > MaxLeftTime)
{
return;
}
var dyk = new DYK.DidYouKnow(wiki);
var draft = dyk.PopDraft(nextIssueDate);
dyk.ArchiveDraftTalk(prevIssueDate);
dyk.ArchiveCurrent(prevIssueDate, nextIssueDate);
dyk.SetCurrent(draft);
dyk.RemoveMarkedFromNextIssue(nextIssueDate);
dyk.RemoveFromPreparationTimetable(nextIssueDate);
var issueValidUntil = nextIssueDate.AddDays(DYKPeriod);
dyk.Stabilize(draft, new DateTimeOffset(issueValidUntil, TimeZone.GetUtcOffset(issueValidUntil)));
}
public async Task <JObject> LoadAsync(MediaWiki wiki, string title)
{
var results = (await Task.WhenAll(_requests.Values.Select(args => {
var obj = new JObject();
if (args["_title"] != null)
{
obj[args.Value <string>("_title")] = title;
}
obj.Merge(args);
return(obj);
}).Select(wiki.Exec)))
.Zip(_requests, (res, req) => new { Key = req.Key, Value = res })
.ToDictionary(x => x.Key, x => x.Value);
var result = new JObject();
foreach (var cb in _callbacks)
{
cb(results, result);
}
return(result);
}
private void Stabilize(MediaWiki wiki, string logTitle, LastDateChecked last)
{
var lines = ParseLog(wiki.GetPage(logTitle));
var lastDate = last.Get() ?? DateTimeOffset.MinValue;
// checking for logDate >= lastDate because log date has minute resolution
foreach (var page in lines.Where(l => l.Date >= lastDate).SelectMany(l => l.Titles).Select(MediaWiki.UnscapeTitle).Distinct())
{
StabilizeArticle(wiki, page);
}
var nextDate = lines.Max(l => l.Date);
// If the last log entry was long enough ago, we can safely move the pointer one
// minute forward. This way we won't have to check the same entry next time.
if (nextDate.AddMinutes(2) < DateTimeOffset.UtcNow)
{
nextDate = nextDate.AddMinutes(1);
}
last.Set(nextDate);
}
public static void Main(string[] args)
{
var wiki = new MediaWiki();
var result = wiki.Search("HelloWorld");
foreach (var search in result.query.search)
{
Console.WriteLine(search.title);
Console.WriteLine(search.snippet);
Console.WriteLine(search.timestamp);
}
Console.WriteLine();
wiki.Language = Languages.Chinese;
result = wiki.SearchAsync(new SearchRequest("我爱你")
{
srlimit = 3
}).GetAwaiter().GetResult();
foreach (var search in result.query.search)
{
Console.WriteLine(search.title);
Console.WriteLine(search.snippet);
Console.WriteLine(search.timestamp);
}
}
public void Execute(MediaWiki wiki, string[] commandLine)
{
Stabilize(wiki, "Шаблон:Актуальные события", DateTimeOffset.Now.AddDays(7));
}
