public void ProcessPage(InputParameters inp) { var title = inp.Fromstr; if (string.IsNullOrEmpty(title)) { _commonlog.LogData("Incorrect page title. Finishing.", 5); return; } LogIn(); title = HttpUtility.UrlDecode(title).Replace('_', ' '); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); _commonlog.LogData("", 5); _commonlog.LogData("One page processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); if (inp.Fullcheck) { _commonlog.LogData("Full check", 4); } TryProcessPortion(inp.Langcode, inp.Projectcode, new List <string> { title }, inp.Fullcheck, inp.OnlyUpdate); }
public void ProcessNewPages(InputParameters inp) { _commonlog.LogData("", 5); _commonlog.LogData("New page processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Hours:", inp.Hours.ToString(), 4); _commonlog.LogData("Query size:", inp.Query.ToString(), 4); if (inp.Fullcheck) { _commonlog.LogData("Full check", 4); } var pages = new List <PageInfo>(); Dictionary <string, object> queryParams = new Dictionary <string, object> { { "querytype", "newpages" }, { "ns", inp.Ns }, { "limit", inp.Query }, { "withcomments", 1 }, { "hours", inp.Hours }, { "timefiltration", 1 } }; WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); _commonlog.LogData("Found new page(s):", pages.Count.ToString(), 4); if (pages.Count <= 0) { _commonlog.LogData("Finishing", 4); return; } LogIn(); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); int portionNumber = (pages.Count - 1) / 500 + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * 500; var num = pages.Count - firstind; if (num > 500) { num = 500; } _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); TryProcessPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(), inp.Fullcheck, inp.OnlyUpdate); } }
public void ProcessCategoryPages(InputParameters inp) { LogIn(); var catname = inp.Catname; catname = HttpUtility.UrlDecode(catname).Replace('_', ' '); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); _commonlog.LogData("", 5); _commonlog.LogData("Category processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Category:", catname, 4); _commonlog.LogData("Depth:", inp.Depth.ToString(), 4); if (inp.Fullcheck) { _commonlog.LogData("Full check", 4); } var pages = new List <PageInfo>(); pages.Clear(); Dictionary <string, object> queryParams = new Dictionary <string, object> { { "querytype", "categorytree" }, { "ns", inp.Ns }, { "limit", inp.Query }, { "categoryname", catname }, { "depth", inp.Depth } }; WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); int portionNumber = (pages.Count - 1) / _bigPortionSize + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * _bigPortionSize; var num = pages.Count - firstind; if (num > _bigPortionSize) { num = _bigPortionSize; } _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); TryProcessPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(), inp.Fullcheck, inp.OnlyUpdate); } }
public void ProcessRangePages(InputParameters inp) { LogIn(); var fromStr = inp.Fromstr; var toStr = inp.Tostr; if (string.IsNullOrEmpty(fromStr)) { fromStr = "!"; } if (string.IsNullOrEmpty(toStr)) { toStr = ""; } fromStr = HttpUtility.UrlDecode(fromStr).Replace('_', ' '); toStr = HttpUtility.UrlDecode(toStr).Replace('_', ' '); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); _commonlog.LogData("", 5); _commonlog.LogData("Range processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Starting:", fromStr, 4); if (toStr.Length > 0) { _commonlog.LogData("Ending:", toStr, 4); } if (inp.Fullcheck) { _commonlog.LogData("Full check", 4); } var pages = new List <PageInfo>(); while (true) { pages.Clear(); Dictionary <string, object> queryParams = new Dictionary <string, object> { { "querytype", "allpages" }, { "ns", inp.Ns }, { "limit", inp.Query }, { "offset", fromStr } }; var newFromStr = WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); int portionNumber = (pages.Count - 1) / _bigPortionSize + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * _bigPortionSize; var num = pages.Count - firstind; if (num > _bigPortionSize) { num = _bigPortionSize; } _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); TryProcessPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(), inp.Fullcheck, inp.OnlyUpdate); } if (newFromStr.Length == 0) { break; } if (toStr.Length > 0 && newFromStr.CompareTo(toStr) > 0) { break; } fromStr = newFromStr; } }
public void ProcessRedirectRange(InputParameters inp) { LogIn(); var fromStr = inp.Fromstr; var toStr = inp.Tostr; if (string.IsNullOrEmpty(fromStr)) fromStr = "!"; if (string.IsNullOrEmpty(toStr)) toStr = ""; // fromStr = HttpUtility.UrlDecode(fromStr).Replace('_', ' '); // toStr = HttpUtility.UrlDecode(toStr).Replace('_', ' '); // fromStr = "B"; _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); _botConfiguration.GetNamespaces(inp.Langcode, inp.Projectcode); _commonlog.LogData("", 5); _commonlog.LogData("Range processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Starting:", fromStr, 4); if (toStr.Length > 0) _commonlog.LogData("Ending:", toStr, 4); var pages = new List<PageInfo>(); while (true) { pages.Clear(); Dictionary<string, object> queryParams = new Dictionary<string, object> { {"querytype", "allpages"}, {"ns", inp.Ns}, {"limit", inp.Query}, {"offset", fromStr}, {"redirtype", "redirects"} }; var newFromStr = WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); int portionNumber = (pages.Count - 1) / _bigPortionSize + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * _bigPortionSize; var num = pages.Count - firstind; if (num > _bigPortionSize) num = _bigPortionSize; _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); ProcessMovedPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).ToList()); } if (newFromStr.Length == 0) break; if (toStr.Length > 0 && newFromStr.CompareTo(toStr) > 0) break; fromStr = newFromStr; } }
public void FindMovedCategories(InputParameters inp) { var newCats = new List<PageInfo>(); Dictionary<string, object> queryParams = new Dictionary<string, object> { {"querytype", "newpages"}, {"ns", 14}, {"limit", inp.Query}, {"withcomments", 1}, {"hours", inp.Hours} }; WikiApiFunctions.FillFromApiQuery(newCats, inp.Langcode, inp.Projectcode, queryParams); if (newCats.Count == 0) return; LogIn(); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); _botConfiguration.GetNamespaces(inp.Langcode, inp.Projectcode); GetCategoryRedirectTemplate(inp.Langcode, inp.Projectcode); if (inp.Mode == "all" || inp.Mode == "talkmove") { ProcessMovedCategoryTalks(inp, newCats); } if (inp.Mode == "all" || inp.Mode == "catredir") { GetCategoryRedirectTemplate(inp.Langcode, inp.Projectcode); if (_categoryRedirectTemplate.Length != 0) { var wikidataLinks = GetWikidataLinks(inp.Langcode, inp.Projectcode, newCats.Select(p => p.Title).ToList()); newCats = newCats.Where(p => !wikidataLinks.ContainsKey(p.Title)).ToList(); ProcessCategoryRedirects(inp, newCats); } } if (inp.Mode == "all" || inp.Mode == "talkmovedtemplate") { GetCategoryTalkMovedTemplate(inp.Langcode, inp.Projectcode); if (_categoryTalkMovedTemplate.Length != 0) { var wikidataLinks = GetWikidataLinks(inp.Langcode, inp.Projectcode, newCats.Select(p => p.Title).ToList()); newCats = newCats.Where(p => !wikidataLinks.ContainsKey(p.Title)).ToList(); ProcessCategoryTalkMovedTemplates(inp, newCats); } } if (inp.Mode == "all" || inp.Mode == "replacedcategory") { var wikidataLinks = GetWikidataLinks(inp.Langcode, inp.Projectcode, newCats.Select(p => p.Title).ToList()); newCats = newCats.Where(p => !wikidataLinks.ContainsKey(p.Title)).ToList(); ProcessReplacedCategories(inp, newCats); } }
public void ProcessCategoryRedirectRange2(InputParameters inp) { _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); GetCategoryRedirectTemplate(inp.Langcode, inp.Projectcode); _commonlog.LogData("", 5); _commonlog.LogData("Range processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); if (_categoryRedirectTemplate.Length == 0) { _commonlog.LogData("There is no category redirect template. Finishing", 4); return; } var pages = new List<PageInfo>(); Dictionary<string, object> queryParams = new Dictionary<string, object> { {"querytype", "transclusions"}, {"ns", 14}, {"limit", inp.Query}, {"title", _categoryRedirectTemplate}, {"getall", 1} }; WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); LogIn(); int portionNumber = (pages.Count - 1) / 500 + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * 500; var num = pages.Count - firstind; if (num > 500) num = 500; _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); ProcessCategoryRedirects(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).Select(p => p.Title).ToList()); } }
public void ProcessMovedPages(InputParameters inp) { _commonlog.LogData("", 5); _commonlog.LogData("move page processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Hours:", inp.Hours.ToString(), 4); _commonlog.LogData("Query size:", inp.Query.ToString(), 4); var pages = new List<PageInfo>(); var timeBorder = DateTime.UtcNow.AddMinutes(-_botConfiguration.MoveWaitMin); Dictionary<string, object> queryParams = new Dictionary<string, object> { {"querytype", "log"}, {"ns", 14}, {"limit", inp.Query}, {"type", "move"}, {"hours", inp.Hours}, {"timefiltration", 1} }; WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); _botConfiguration.GetNamespaces(inp.Langcode, inp.Projectcode); pages = pages.Where(pi => pi.Timestamp.CompareTo(timeBorder) <= 0).ToList(); pages = _botConfiguration.FilterByNamespace(pages, inp.Langcode, -100); _commonlog.LogData("Found new page(s):", pages.Count.ToString(), 4); if (pages.Count <= 0) { _commonlog.LogData("Finishing", 4); return; } LogIn(); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); int portionNumber = (pages.Count - 1) / 500 + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * 500; var num = pages.Count - firstind; if (num > 500) num = 500; _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); ProcessMovedPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).ToList()); } }
public void ProcessPage(InputParameters inp) { var title = inp.Fromstr; if (string.IsNullOrEmpty(title)) { _commonlog.LogData("Incorrect page title. Finishing.", 5); return; } LogIn(); title = HttpUtility.UrlDecode(title).Replace('_', ' '); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); _commonlog.LogData("", 5); _commonlog.LogData("One page processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); if (inp.Fullcheck) _commonlog.LogData("Full check", 4); TryProcessPortion(inp.Langcode, inp.Projectcode, new List<string> { title }, inp.Fullcheck, inp.OnlyUpdate); }
public void ProcessUserContributions(InputParameters inp) { _commonlog.LogData("", 5); _commonlog.LogData("User contribution processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("User:"******"Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Hours:", inp.Hours.ToString(), 4); _commonlog.LogData("Query size:", inp.Query.ToString(), 4); if (inp.Fullcheck) _commonlog.LogData("Full check", 4); var pages = new List<PageInfo>(); Dictionary<string, object> queryParams = new Dictionary<string, object> { {"querytype", "usercontributions"}, {"ns", inp.Ns}, {"limit", inp.Query}, {"timefiltration", 1} }; WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); _commonlog.LogData("Found new page(s):", pages.Count.ToString(), 4); if (pages.Count <= 0) { _commonlog.LogData("Finishing", 4); return; } LogIn(); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); int portionNumber = (pages.Count - 1) / 500 + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * 500; var num = pages.Count - firstind; if (num > 500) num = 500; _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); TryProcessPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(), inp.Fullcheck, inp.OnlyUpdate); } }
public void ProcessCategoryPages(InputParameters inp) { LogIn(); var catname = inp.Catname; catname = HttpUtility.UrlDecode(catname).Replace('_', ' '); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); _commonlog.LogData("", 5); _commonlog.LogData("Category processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Category:", catname, 4); _commonlog.LogData("Depth:", inp.Depth.ToString(), 4); if (inp.Fullcheck) _commonlog.LogData("Full check", 4); var pages = new List<PageInfo>(); pages.Clear(); Dictionary<string, object> queryParams = new Dictionary<string, object> { {"querytype", "categorytree"}, {"ns", inp.Ns}, {"limit", inp.Query}, {"categoryname", catname}, {"depth", inp.Depth} }; WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); int portionNumber = (pages.Count - 1) / _bigPortionSize + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * _bigPortionSize; var num = pages.Count - firstind; if (num > _bigPortionSize) num = _bigPortionSize; _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); TryProcessPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(), inp.Fullcheck, inp.OnlyUpdate); } }
public LanguageCodes GetWikiCodes(string project) { _languageCodes = new LanguageCodes(project); _languageCodes.GetWikiCodes(); return(_languageCodes); }
public LanguageCodes GetWikiCodes() { _languageCodes = new LanguageCodes(); _languageCodes.GetWikiCodes(); return(_languageCodes); }
public LanguageCodes GetWikiCodes(string project) { _languageCodes = new LanguageCodes(project); _languageCodes.GetWikiCodes(); return _languageCodes; }
public LanguageCodes GetWikiCodes() { _languageCodes = new LanguageCodes(); _languageCodes.GetWikiCodes(); return _languageCodes; }