public void ProcessNewPages(InputParameters inp)
{
_commonlog.LogData("", 5);
_commonlog.LogData("New page processing", 4);
_commonlog.LogData("Language:", inp.Langcode, 4);
_commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
_commonlog.LogData("Hours:", inp.Hours.ToString(), 4);
_commonlog.LogData("Query size:", inp.Query.ToString(), 4);
if (inp.Fullcheck)
{
_commonlog.LogData("Full check", 4);
}
var pages = new List <PageInfo>();
Dictionary <string, object> queryParams = new Dictionary <string, object>
{
{ "querytype", "newpages" },
{ "ns", inp.Ns },
{ "limit", inp.Query },
{ "withcomments", 1 },
{ "hours", inp.Hours },
{ "timefiltration", 1 }
};
WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
_commonlog.LogData("Found new page(s):", pages.Count.ToString(), 4);
if (pages.Count <= 0)
{
_commonlog.LogData("Finishing", 4);
return;
}
LogIn();
_wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);
int portionNumber = (pages.Count - 1) / 500 + 1;
for (int i = 0; i < portionNumber; i++)
{
var firstind = i * 500;
var num = pages.Count - firstind;
if (num > 500)
{
num = 500;
}
_commonlog.LogData("", 2);
_commonlog.LogData("New portion of pages", 2);
_commonlog.LogData("Number of pages:", num.ToString(), 2);
TryProcessPortion(inp.Langcode,
inp.Projectcode,
pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(),
inp.Fullcheck,
inp.OnlyUpdate);
}
}
private void GetLocalInterwiki(List <MultilingualPage> mlpl, string code, string project, string[] titles)
{
var allPagesInterwikis = WikiApiFunctions.GetLocalInterwiki(code, project, titles);
var newArr = new Dictionary <string, Dictionary <string, string> >();
foreach (var kvv in allPagesInterwikis)
{
var newIw = new Dictionary <string, string>();
foreach (var kv in kvv.Value)
{
var title = kv.Value;
if (title.StartsWith("voy:"))
{
title = title.Substring(4);
}
newIw.Add(kv.Key, title);
}
newArr.Add(kvv.Key, newIw);
}
allPagesInterwikis = newArr;
foreach (var mlp in mlpl)
{
foreach (var mlpInterwiki in mlp.Interwikis.Where(iw => iw.Code == code && allPagesInterwikis.ContainsKey(iw.Title)).ToList())
{
mlpInterwiki.IsLocalIwChecked = true;
var localPageInterwikis = allPagesInterwikis[mlpInterwiki.Title];
foreach (var iw in localPageInterwikis.Where(iw => !mlp.Interwikis.Any(mlpIw => mlpIw.Code == iw.Key && mlpIw.Title == iw.Value)).ToList())
{
if (!_wikiCodes.ContainsLanguageCode(iw.Key))
{
continue;
}
mlp.Interwikis.Add(new InterwikiItem
{
Code = iw.Key,
Title = iw.Value,
IsToSection = iw.Value.Contains("#")
});
}
}
//Esperanto API bug
if (code == "eo")
{
foreach (var mlpInterwiki in mlp.Interwikis.Where(iw => iw.Code == "eo" && !iw.IsLocalIwChecked && titles.Contains(iw.Title)).ToList())
{
mlpInterwiki.IsLocalIwChecked = true;
}
}
}
}
public void ProcessCategoryPages(InputParameters inp)
{
LogIn();
var catname = inp.Catname;
catname = HttpUtility.UrlDecode(catname).Replace('_', ' ');
_wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);
_commonlog.LogData("", 5);
_commonlog.LogData("Category processing", 4);
_commonlog.LogData("Language:", inp.Langcode, 4);
_commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
_commonlog.LogData("Category:", catname, 4);
_commonlog.LogData("Depth:", inp.Depth.ToString(), 4);
if (inp.Fullcheck)
{
_commonlog.LogData("Full check", 4);
}
var pages = new List <PageInfo>();
pages.Clear();
Dictionary <string, object> queryParams = new Dictionary <string, object>
{
{ "querytype", "categorytree" },
{ "ns", inp.Ns },
{ "limit", inp.Query },
{ "categoryname", catname },
{ "depth", inp.Depth }
};
WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
int portionNumber = (pages.Count - 1) / _bigPortionSize + 1;
for (int i = 0; i < portionNumber; i++)
{
var firstind = i * _bigPortionSize;
var num = pages.Count - firstind;
if (num > _bigPortionSize)
{
num = _bigPortionSize;
}
_commonlog.LogData("", 2);
_commonlog.LogData("New portion of pages", 2);
_commonlog.LogData("Number of pages:", num.ToString(), 2);
TryProcessPortion(inp.Langcode,
inp.Projectcode,
pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(),
inp.Fullcheck,
inp.OnlyUpdate);
}
}
public void GetNamespaces(string code, string project)
{
if (Namespaces == null)
{
Namespaces = new Dictionary <string, Dictionary <int, string> >();
}
if (Namespaces.ContainsKey(code))
{
return;
}
Namespaces.Add(code, WikiApiFunctions.GetNamespaces(code, project));
}
private bool GetWikiCodesFromSite(string projectUrl)
{
for (int i = 0; i < 5; i++)
{
var wikiCodes = WikiApiFunctions.GetWikiCodes(projectUrl);
if (wikiCodes.ContainsKey("wikipedia") && wikiCodes["wikipedia"].Count > 10)
{
WikiCodes = wikiCodes;
return(true);
}
System.Threading.Thread.Sleep(i * 1000);
}
return(false);
}
private void GetWikidataLinks(List <MultilingualPage> mlpl, string code, string project, string[] titles)
{
var wdLangCode = _wikiCodes.ToProjectCode(code);
var wikiDataLinks = WikiApiFunctions.GetWikidataLinks(wdLangCode, titles, true);
foreach (var title in titles)
{
foreach (var interwikiItem in mlpl.SelectMany(mlp => mlp.Interwikis).Where(iw => iw.Title == title && iw.Code == code).ToList())
{
interwikiItem.IsWdChecked = true;
}
}
foreach (var wdEntities in wikiDataLinks)
{
if (wdEntities.Count == 0)
{
continue;
}
var curProjectLinks = _wikiCodes.ToLanguageCodes(wdEntities);
curProjectLinks.Add("wikidata", wdEntities["wikidata"]);
foreach (var mlp in mlpl)
{
if (!mlp.Interwikis.Any(iw => iw.Title == curProjectLinks[code] && iw.Code == code))
{
continue;
}
var wdId = wdEntities["wikidata"];
if (mlp.IsOnWikiData && mlp.WikiDataItem != wdId)
{
mlp.HasConflict = true;
mlp.ConflictDescription =
string.Format("Conflict. Starting from: {0}:{1}. Found items on WikiData: {2} and {3}",
mlp.Interwikis[0].Code,
mlp.Interwikis[0].Title, mlp.WikiDataItem, wdId);
continue;
}
mlp.IsOnWikiData = true;
mlp.WikiDataItem = wdId;
foreach (var wdIw in curProjectLinks)
{
if (wdIw.Key == "wikidata")
{
continue;
}
if (mlp.Interwikis.Any(iw => iw.Code == wdIw.Key && iw.Title == wdIw.Value))
{
continue;
}
mlp.Interwikis.Add(new InterwikiItem {
Code = wdIw.Key, Title = wdIw.Value
});
}
foreach (var iw in mlp.Interwikis.Where(iw => curProjectLinks.ContainsKey(iw.Code) && curProjectLinks[iw.Code] == iw.Title).ToList())
{
iw.IsWdChecked = true;
iw.IsOnWd = true;
}
}
}
}
private void GetProperTitles(List <MultilingualPage> mlpl, string code, string project, string[] titles)
{
var pages = WikiApiFunctions.GetProperTitles(code, project, titles);
foreach (var title in titles)
{
var page = pages.FirstOrDefault(p => p.Title == title);
if (page == null)
{
page = WikiApiFunctions.GetProperTitle(code, project, title);
}
foreach (var mlp in mlpl)
{
var interwiki = mlp.Interwikis.FirstOrDefault(iw => iw.Code == code && iw.Title == title);
if (interwiki == null)
{
continue;
}
interwiki.Info = page;
interwiki.IsExcluded = page.IsMissing;
interwiki.IsChecked = true;
if (page.RedirectTo.Length <= 0 && page.NormalizedTitle.Length <= 0)
{
continue;
}
interwiki.IsRedirect = true;
var redirectedIw = new InterwikiItem {
Code = code
};
if (page.RedirectTo.Length > 0)
{
interwiki.RedirectTo = page.RedirectTo;
if (page.RedirectionSection.Length > 0)
{
interwiki.RedirectTo += "#" + page.RedirectionSection;
}
redirectedIw.Title = page.RedirectTo;
}
else
{
interwiki.RedirectTo = page.NormalizedTitle;
redirectedIw.Title = page.NormalizedTitle;
}
if (mlp.Interwikis.Any(iw => iw.Code == code && iw.Title == redirectedIw.Title))
{
continue;
}
var redirectedPage = pages.FirstOrDefault(p => p.Title == redirectedIw.Title);
if (redirectedPage == null)
{
redirectedIw.IsExcluded = true;
}
else
{
redirectedIw.Info = redirectedPage;
redirectedIw.IsExcluded = redirectedPage.IsMissing;
}
redirectedIw.IsRedirect = false;
redirectedIw.IsChecked = true;
mlp.Interwikis.Add(redirectedIw);
}
}
}
public void ProcessRangePages(InputParameters inp)
{
LogIn();
var fromStr = inp.Fromstr;
var toStr = inp.Tostr;
if (string.IsNullOrEmpty(fromStr))
{
fromStr = "!";
}
if (string.IsNullOrEmpty(toStr))
{
toStr = "";
}
fromStr = HttpUtility.UrlDecode(fromStr).Replace('_', ' ');
toStr = HttpUtility.UrlDecode(toStr).Replace('_', ' ');
_wikiCodes = _botConfiguration.GetWikiCodes(inp.Projectcode);
_commonlog.LogData("", 5);
_commonlog.LogData("Range processing", 4);
_commonlog.LogData("Language:", inp.Langcode, 4);
_commonlog.LogData("Namespace:", inp.Ns.ToString(), 4);
_commonlog.LogData("Starting:", fromStr, 4);
if (toStr.Length > 0)
{
_commonlog.LogData("Ending:", toStr, 4);
}
if (inp.Fullcheck)
{
_commonlog.LogData("Full check", 4);
}
var pages = new List <PageInfo>();
while (true)
{
pages.Clear();
Dictionary <string, object> queryParams = new Dictionary <string, object>
{
{ "querytype", "allpages" },
{ "ns", inp.Ns },
{ "limit", inp.Query },
{ "offset", fromStr }
};
var newFromStr = WikiApiFunctions.FillFromApiQuery(pages, inp.Langcode, inp.Projectcode, queryParams);
int portionNumber = (pages.Count - 1) / _bigPortionSize + 1;
for (int i = 0; i < portionNumber; i++)
{
var firstind = i * _bigPortionSize;
var num = pages.Count - firstind;
if (num > _bigPortionSize)
{
num = _bigPortionSize;
}
_commonlog.LogData("", 2);
_commonlog.LogData("New portion of pages", 2);
_commonlog.LogData("Number of pages:", num.ToString(), 2);
TryProcessPortion(inp.Langcode,
inp.Projectcode,
pages.Skip(firstind).Take(num).Select(p => p.Title).ToList(),
inp.Fullcheck,
inp.OnlyUpdate);
}
if (newFromStr.Length == 0)
{
break;
}
if (toStr.Length > 0 && newFromStr.CompareTo(toStr) > 0)
{
break;
}
fromStr = newFromStr;
}
}
