public void ProcessNewPages(InputParameters inp) { _commonlog.LogData("", 5); _commonlog.LogData("New page processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Hours:", inp.Hours.ToString(), 4); _commonlog.LogData("Query size:", inp.Query.ToString(), 4); if (inp.Fullcheck) { _commonlog.LogData("Full check", 4); } var pages = new List <PageInfo>(); Dictionary <string, object> queryParams = new Dictionary <string, object> { { "querytype", "newpages" }, { "ns", inp.Ns }, { "limit", inp.Query }, { "withcomments", 1 }, { "hours", inp.Hours }, { "timefiltration", 1 } }; WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); _commonlog.LogData("Found new page(s):", pages.Count.ToString(), 4); if (pages.Count <= 0) { _commonlog.LogData("Finishing", 4); return; } LogIn(); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); int portionNumber = (pages.Count - 1) / 500 + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * 500; var num = pages.Count - firstind; if (num > 500) { num = 500; } _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); TryProcessPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(), inp.Fullcheck, inp.OnlyUpdate); } }
public void ProcessCategoryPages(InputParameters inp) { LogIn(); var catname = inp.Catname; catname = HttpUtility.UrlDecode(catname).Replace('_', ' '); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); _commonlog.LogData("", 5); _commonlog.LogData("Category processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Category:", catname, 4); _commonlog.LogData("Depth:", inp.Depth.ToString(), 4); if (inp.Fullcheck) { _commonlog.LogData("Full check", 4); } var pages = new List <PageInfo>(); pages.Clear(); Dictionary <string, object> queryParams = new Dictionary <string, object> { { "querytype", "categorytree" }, { "ns", inp.Ns }, { "limit", inp.Query }, { "categoryname", catname }, { "depth", inp.Depth } }; WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); int portionNumber = (pages.Count - 1) / _bigPortionSize + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * _bigPortionSize; var num = pages.Count - firstind; if (num > _bigPortionSize) { num = _bigPortionSize; } _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); TryProcessPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(), inp.Fullcheck, inp.OnlyUpdate); } }
public void ProcessRangePages(InputParameters inp) { LogIn(); var fromStr = inp.Fromstr; var toStr = inp.Tostr; if (string.IsNullOrEmpty(fromStr)) { fromStr = "!"; } if (string.IsNullOrEmpty(toStr)) { toStr = ""; } fromStr = HttpUtility.UrlDecode(fromStr).Replace('_', ' '); toStr = HttpUtility.UrlDecode(toStr).Replace('_', ' '); _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode); _commonlog.LogData("", 5); _commonlog.LogData("Range processing", 4); _commonlog.LogData("Language:", inp.Langcode, 4); _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4); _commonlog.LogData("Starting:", fromStr, 4); if (toStr.Length > 0) { _commonlog.LogData("Ending:", toStr, 4); } if (inp.Fullcheck) { _commonlog.LogData("Full check", 4); } var pages = new List <PageInfo>(); while (true) { pages.Clear(); Dictionary <string, object> queryParams = new Dictionary <string, object> { { "querytype", "allpages" }, { "ns", inp.Ns }, { "limit", inp.Query }, { "offset", fromStr } }; var newFromStr = WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams); int portionNumber = (pages.Count - 1) / _bigPortionSize + 1; for (int i = 0; i < portionNumber; i++) { var firstind = i * _bigPortionSize; var num = pages.Count - firstind; if (num > _bigPortionSize) { num = _bigPortionSize; } _commonlog.LogData("", 2); _commonlog.LogData("New portion of pages", 2); _commonlog.LogData("Number of pages:", num.ToString(), 2); TryProcessPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(), inp.Fullcheck, inp.OnlyUpdate); } if (newFromStr.Length == 0) { break; } if (toStr.Length > 0 && newFromStr.CompareTo(toStr) > 0) { break; } fromStr = newFromStr; } }