Ejemplo n.º 1
0
        public void ProcessPage(InputParameters inp)
        {
            var title = inp.Fromstr;

            if (string.IsNullOrEmpty(title))
            {
                _commonlog.LogData("Incorrect page title. Finishing.", 5);
                return;
            }
            LogIn();

            title      = HttpUtility.UrlDecode(title).Replace('_', ' ');
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);

            _commonlog.LogData("", 5);
            _commonlog.LogData("One page processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
            if (inp.Fullcheck)
            {
                _commonlog.LogData("Full check", 4);
            }

            TryProcessPortion(inp.Langcode,
                              inp.Projectcode,
                              new List <string> {
                title
            },
                              inp.Fullcheck,
                              inp.OnlyUpdate);
        }
Ejemplo n.º 2
0
        public void ProcessNewPages(InputParameters inp)
        {
            _commonlog.LogData("", 5);
            _commonlog.LogData("New page processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
            _commonlog.LogData("Hours:", inp.Hours.ToString(), 4);
            _commonlog.LogData("Query size:", inp.Query.ToString(), 4);
            if (inp.Fullcheck)
            {
                _commonlog.LogData("Full check", 4);
            }

            var pages = new List <PageInfo>();
            Dictionary <string, object> queryParams = new Dictionary <string, object>
            {
                { "querytype", "newpages" },
                { "ns", inp.Ns },
                { "limit", inp.Query },
                { "withcomments", 1 },
                { "hours", inp.Hours },
                { "timefiltration", 1 }
            };

            WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
            _commonlog.LogData("Found new page(s):", pages.Count.ToString(), 4);
            if (pages.Count <= 0)
            {
                _commonlog.LogData("Finishing", 4);
                return;
            }

            LogIn();
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);

            int portionNumber = (pages.Count - 1) / 500 + 1;

            for (int i = 0; i < portionNumber; i++)
            {
                var firstind = i * 500;
                var num      = pages.Count - firstind;
                if (num > 500)
                {
                    num = 500;
                }
                _commonlog.LogData("", 2);
                _commonlog.LogData("New portion of pages", 2);
                _commonlog.LogData("Number of pages:", num.ToString(), 2);
                TryProcessPortion(inp.Langcode,
                                  inp.Projectcode,
                                  pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(),
                                  inp.Fullcheck,
                                  inp.OnlyUpdate);
            }
        }
Ejemplo n.º 3
0
        public void ProcessCategoryPages(InputParameters inp)
        {
            LogIn();

            var catname = inp.Catname;

            catname    = HttpUtility.UrlDecode(catname).Replace('_', ' ');
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);

            _commonlog.LogData("", 5);
            _commonlog.LogData("Category processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
            _commonlog.LogData("Category:", catname, 4);
            _commonlog.LogData("Depth:", inp.Depth.ToString(), 4);
            if (inp.Fullcheck)
            {
                _commonlog.LogData("Full check", 4);
            }

            var pages = new List <PageInfo>();

            pages.Clear();
            Dictionary <string, object> queryParams = new Dictionary <string, object>
            {
                { "querytype", "categorytree" },
                { "ns", inp.Ns },
                { "limit", inp.Query },
                { "categoryname", catname },
                { "depth", inp.Depth }
            };

            WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
            int portionNumber = (pages.Count - 1) / _bigPortionSize + 1;

            for (int i = 0; i < portionNumber; i++)
            {
                var firstind = i * _bigPortionSize;
                var num      = pages.Count - firstind;
                if (num > _bigPortionSize)
                {
                    num = _bigPortionSize;
                }
                _commonlog.LogData("", 2);
                _commonlog.LogData("New portion of pages", 2);
                _commonlog.LogData("Number of pages:", num.ToString(), 2);
                TryProcessPortion(inp.Langcode,
                                  inp.Projectcode,
                                  pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(),
                                  inp.Fullcheck,
                                  inp.OnlyUpdate);
            }
        }
Ejemplo n.º 4
0
        public void ProcessRangePages(InputParameters inp)
        {
            LogIn();
            var fromStr = inp.Fromstr;
            var toStr   = inp.Tostr;

            if (string.IsNullOrEmpty(fromStr))
            {
                fromStr = "!";
            }
            if (string.IsNullOrEmpty(toStr))
            {
                toStr = "";
            }

            fromStr    = HttpUtility.UrlDecode(fromStr).Replace('_', ' ');
            toStr      = HttpUtility.UrlDecode(toStr).Replace('_', ' ');
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);

            _commonlog.LogData("", 5);
            _commonlog.LogData("Range processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
            _commonlog.LogData("Starting:", fromStr, 4);
            if (toStr.Length > 0)
            {
                _commonlog.LogData("Ending:", toStr, 4);
            }
            if (inp.Fullcheck)
            {
                _commonlog.LogData("Full check", 4);
            }

            var pages = new List <PageInfo>();

            while (true)
            {
                pages.Clear();
                Dictionary <string, object> queryParams = new Dictionary <string, object>
                {
                    { "querytype", "allpages" },
                    { "ns", inp.Ns },
                    { "limit", inp.Query },
                    { "offset", fromStr }
                };
                var newFromStr    = WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
                int portionNumber = (pages.Count - 1) / _bigPortionSize + 1;
                for (int i = 0; i < portionNumber; i++)
                {
                    var firstind = i * _bigPortionSize;
                    var num      = pages.Count - firstind;
                    if (num > _bigPortionSize)
                    {
                        num = _bigPortionSize;
                    }
                    _commonlog.LogData("", 2);
                    _commonlog.LogData("New portion of pages", 2);
                    _commonlog.LogData("Number of pages:", num.ToString(), 2);
                    TryProcessPortion(inp.Langcode,
                                      inp.Projectcode,
                                      pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(),
                                      inp.Fullcheck,
                                      inp.OnlyUpdate);
                }

                if (newFromStr.Length == 0)
                {
                    break;
                }
                if (toStr.Length > 0 && newFromStr.CompareTo(toStr) > 0)
                {
                    break;
                }
                fromStr = newFromStr;
            }
        }
Ejemplo n.º 5
0
        public void ProcessRedirectRange(InputParameters inp)
        {
            LogIn();
            var fromStr = inp.Fromstr;
            var toStr = inp.Tostr;

            if (string.IsNullOrEmpty(fromStr))
                fromStr = "!";
            if (string.IsNullOrEmpty(toStr))
                toStr = "";

            //    fromStr = HttpUtility.UrlDecode(fromStr).Replace('_', ' ');
            //    toStr = HttpUtility.UrlDecode(toStr).Replace('_', ' ');
            //    fromStr = "B";
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);
            _botConfiguration.GetNamespaces(inp.Langcode, inp.Projectcode);

            _commonlog.LogData("", 5);
            _commonlog.LogData("Range processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
            _commonlog.LogData("Starting:", fromStr, 4);
            if (toStr.Length > 0)
                _commonlog.LogData("Ending:", toStr, 4);

            var pages = new List<PageInfo>();

            while (true)
            {
                pages.Clear();

                Dictionary<string, object> queryParams = new Dictionary<string, object>
                    {
                        {"querytype", "allpages"},
                        {"ns", inp.Ns},
                        {"limit", inp.Query},
                        {"offset", fromStr},
                        {"redirtype", "redirects"}
                    };
                var newFromStr = WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
                int portionNumber = (pages.Count - 1) / _bigPortionSize + 1;
                for (int i = 0; i < portionNumber; i++)
                {
                    var firstind = i * _bigPortionSize;
                    var num = pages.Count - firstind;
                    if (num > _bigPortionSize)
                        num = _bigPortionSize;
                    _commonlog.LogData("", 2);
                    _commonlog.LogData("New portion of pages", 2);
                    _commonlog.LogData("Number of pages:", num.ToString(), 2);
                    ProcessMovedPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).ToList());
                }

                if (newFromStr.Length == 0)
                    break;
                if (toStr.Length > 0 && newFromStr.CompareTo(toStr) > 0)
                    break;
                fromStr = newFromStr;
            }
        }
Ejemplo n.º 6
0
        public void FindMovedCategories(InputParameters inp)
        {
            var newCats = new List<PageInfo>();
            Dictionary<string, object> queryParams = new Dictionary<string, object>
                {
                    {"querytype", "newpages"},
                    {"ns", 14},
                    {"limit", inp.Query},
                    {"withcomments", 1},
                    {"hours", inp.Hours}
                };

            WikiApiFunctions.FillFromApiQuery(newCats, inp.Langcode, inp.Projectcode, queryParams);

            if (newCats.Count == 0)
                return;

            LogIn();
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);
            _botConfiguration.GetNamespaces(inp.Langcode, inp.Projectcode);
            GetCategoryRedirectTemplate(inp.Langcode, inp.Projectcode);

            if (inp.Mode == "all" || inp.Mode == "talkmove")
            {
                ProcessMovedCategoryTalks(inp, newCats);
            }

            if (inp.Mode == "all" || inp.Mode == "catredir")
            {
                GetCategoryRedirectTemplate(inp.Langcode, inp.Projectcode);
                if (_categoryRedirectTemplate.Length != 0)
                {
                    var wikidataLinks =
                        GetWikidataLinks(inp.Langcode, inp.Projectcode, newCats.Select(p => p.Title).ToList());
                    newCats = newCats.Where(p => !wikidataLinks.ContainsKey(p.Title)).ToList();

                    ProcessCategoryRedirects(inp, newCats);
                }
            }

            if (inp.Mode == "all" || inp.Mode == "talkmovedtemplate")
            {
                GetCategoryTalkMovedTemplate(inp.Langcode, inp.Projectcode);
                if (_categoryTalkMovedTemplate.Length != 0)
                {
                    var wikidataLinks =
                        GetWikidataLinks(inp.Langcode, inp.Projectcode, newCats.Select(p => p.Title).ToList());
                    newCats = newCats.Where(p => !wikidataLinks.ContainsKey(p.Title)).ToList();

                    ProcessCategoryTalkMovedTemplates(inp, newCats);
                }
            }

            if (inp.Mode == "all" || inp.Mode == "replacedcategory")
            {
                var wikidataLinks =
                    GetWikidataLinks(inp.Langcode, inp.Projectcode, newCats.Select(p => p.Title).ToList());
                newCats = newCats.Where(p => !wikidataLinks.ContainsKey(p.Title)).ToList();

                ProcessReplacedCategories(inp, newCats);
            }
        }
Ejemplo n.º 7
0
        public void ProcessCategoryRedirectRange2(InputParameters inp)
        {
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);
            GetCategoryRedirectTemplate(inp.Langcode, inp.Projectcode);

            _commonlog.LogData("", 5);
            _commonlog.LogData("Range processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);

            if (_categoryRedirectTemplate.Length == 0)
            {
                _commonlog.LogData("There is no category redirect template. Finishing", 4);
                return;
            }

            var pages = new List<PageInfo>();
            Dictionary<string, object> queryParams = new Dictionary<string, object>
                {
                    {"querytype", "transclusions"},
                    {"ns", 14},
                    {"limit", inp.Query},
                    {"title", _categoryRedirectTemplate},
                    {"getall", 1}
                };
            WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);

            LogIn();

            int portionNumber = (pages.Count - 1) / 500 + 1;
            for (int i = 0; i < portionNumber; i++)
            {
                var firstind = i * 500;
                var num = pages.Count - firstind;
                if (num > 500)
                    num = 500;
                _commonlog.LogData("", 2);
                _commonlog.LogData("New portion of pages", 2);
                _commonlog.LogData("Number of pages:", num.ToString(), 2);
                ProcessCategoryRedirects(inp.Langcode,
                                         inp.Projectcode,
                                         pages.Skip(firstind).Take(num).Select(p => p.Title).ToList());
            }
        }
Ejemplo n.º 8
0
        public void ProcessMovedPages(InputParameters inp)
        {
            _commonlog.LogData("", 5);
            _commonlog.LogData("move page processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
            _commonlog.LogData("Hours:", inp.Hours.ToString(), 4);
            _commonlog.LogData("Query size:", inp.Query.ToString(), 4);

            var pages = new List<PageInfo>();
            var timeBorder = DateTime.UtcNow.AddMinutes(-_botConfiguration.MoveWaitMin);

            Dictionary<string, object> queryParams = new Dictionary<string, object>
                {
                    {"querytype", "log"},
                    {"ns", 14},
                    {"limit", inp.Query},
                    {"type", "move"},
                    {"hours", inp.Hours},
                    {"timefiltration", 1}
                };
            WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
            _botConfiguration.GetNamespaces(inp.Langcode, inp.Projectcode);
            pages = pages.Where(pi => pi.Timestamp.CompareTo(timeBorder) <= 0).ToList();
            pages = _botConfiguration.FilterByNamespace(pages, inp.Langcode, -100);
            _commonlog.LogData("Found new page(s):", pages.Count.ToString(), 4);
            if (pages.Count <= 0)
            {
                _commonlog.LogData("Finishing", 4);
                return;
            }

            LogIn();
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);

            int portionNumber = (pages.Count - 1) / 500 + 1;
            for (int i = 0; i < portionNumber; i++)
            {
                var firstind = i * 500;
                var num = pages.Count - firstind;
                if (num > 500)
                    num = 500;
                _commonlog.LogData("", 2);
                _commonlog.LogData("New portion of pages", 2);
                _commonlog.LogData("Number of pages:", num.ToString(), 2);
                ProcessMovedPortion(inp.Langcode, inp.Projectcode, pages.Skip(firstind).Take(num).ToList());
            }
        }
Ejemplo n.º 9
0
        public void ProcessPage(InputParameters inp)
        {
            var title = inp.Fromstr;
            if (string.IsNullOrEmpty(title))
            {
                _commonlog.LogData("Incorrect page title. Finishing.", 5);
                return;
            }
            LogIn();

            title = HttpUtility.UrlDecode(title).Replace('_', ' ');
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);

            _commonlog.LogData("", 5);
            _commonlog.LogData("One page processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
            if (inp.Fullcheck)
                _commonlog.LogData("Full check", 4);

            TryProcessPortion(inp.Langcode,
                           inp.Projectcode,
                           new List<string> { title },
                           inp.Fullcheck,
                           inp.OnlyUpdate);
        }
Ejemplo n.º 10
0
        public void ProcessUserContributions(InputParameters inp)
        {
            _commonlog.LogData("", 5);
            _commonlog.LogData("User contribution processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("User:"******"Namespace:", inp.Ns.ToString(), 4);
            _commonlog.LogData("Hours:", inp.Hours.ToString(), 4);
            _commonlog.LogData("Query size:", inp.Query.ToString(), 4);
            if (inp.Fullcheck)
                _commonlog.LogData("Full check", 4);

            var pages = new List<PageInfo>();
            Dictionary<string, object> queryParams = new Dictionary<string, object>
                {
                    {"querytype", "usercontributions"},
                    {"ns", inp.Ns},
                    {"limit", inp.Query},
                    {"timefiltration", 1}
                };

            WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
            _commonlog.LogData("Found new page(s):", pages.Count.ToString(), 4);
            if (pages.Count <= 0)
            {
                _commonlog.LogData("Finishing", 4);
                return;
            }

            LogIn();
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);

            int portionNumber = (pages.Count - 1) / 500 + 1;
            for (int i = 0; i < portionNumber; i++)
            {
                var firstind = i * 500;
                var num = pages.Count - firstind;
                if (num > 500)
                    num = 500;
                _commonlog.LogData("", 2);
                _commonlog.LogData("New portion of pages", 2);
                _commonlog.LogData("Number of pages:", num.ToString(), 2);
                TryProcessPortion(inp.Langcode,
                               inp.Projectcode,
                               pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(),
                               inp.Fullcheck,
                               inp.OnlyUpdate);
            }
        }
Ejemplo n.º 11
0
        public void ProcessCategoryPages(InputParameters inp)
        {
            LogIn();

            var catname = inp.Catname;

            catname = HttpUtility.UrlDecode(catname).Replace('_', ' ');
            _wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);

            _commonlog.LogData("", 5);
            _commonlog.LogData("Category processing", 4);
            _commonlog.LogData("Language:", inp.Langcode, 4);
            _commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
            _commonlog.LogData("Category:", catname, 4);
            _commonlog.LogData("Depth:", inp.Depth.ToString(), 4);
            if (inp.Fullcheck)
                _commonlog.LogData("Full check", 4);

            var pages = new List<PageInfo>();

            pages.Clear();
            Dictionary<string, object> queryParams = new Dictionary<string, object>
                {
                    {"querytype", "categorytree"},
                    {"ns", inp.Ns},
                    {"limit", inp.Query},
                    {"categoryname", catname},
                    {"depth", inp.Depth}
                };
            WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
            int portionNumber = (pages.Count - 1) / _bigPortionSize + 1;
            for (int i = 0; i < portionNumber; i++)
            {
                var firstind = i * _bigPortionSize;
                var num = pages.Count - firstind;
                if (num > _bigPortionSize)
                    num = _bigPortionSize;
                _commonlog.LogData("", 2);
                _commonlog.LogData("New portion of pages", 2);
                _commonlog.LogData("Number of pages:", num.ToString(), 2);
                TryProcessPortion(inp.Langcode,
                               inp.Projectcode,
                               pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(),
                               inp.Fullcheck,
                               inp.OnlyUpdate);
            }
        }
Ejemplo n.º 12
0
 public LanguageCodes GetWikiCodes(string project)
 {
     _languageCodes = new LanguageCodes(project);
     _languageCodes.GetWikiCodes();
     return(_languageCodes);
 }
Ejemplo n.º 13
0
 public LanguageCodes GetWikiCodes()
 {
     _languageCodes = new LanguageCodes();
     _languageCodes.GetWikiCodes();
     return(_languageCodes);
 }
Ejemplo n.º 14
0
 public LanguageCodes GetWikiCodes(string project)
 {
     _languageCodes = new LanguageCodes(project);
     _languageCodes.GetWikiCodes();
     return _languageCodes;
 }
Ejemplo n.º 15
0
 public LanguageCodes GetWikiCodes()
 {
     _languageCodes = new LanguageCodes();
     _languageCodes.GetWikiCodes();
     return _languageCodes;
 }