private void ProcessPortion(string langcode, string projectcode, List <string> pages, bool fullcheck, bool onlyupdate)
{
var mlpl = new List <MultilingualPage>();
foreach (var page in pages)
{
var mlp = new MultilingualPage();
mlp.Interwikis.Add(new InterwikiItem {
Title = page, Code = langcode
});
mlpl.Add(mlp);
}
_commonlog.LogData("First page in the portion:", pages[0], 2);
var logstring = string.Format("Iteration {0}. Check {1} links of language {2}...", 0, pages.Count, langcode);
_commonlog.LogData(logstring, 1);
GetProperTitles(mlpl, langcode, projectcode);
GetWikidataLinks(mlpl, langcode, projectcode);
foreach (var mlp in mlpl.Where(mlp => mlp.IsOnWikiData))
{
foreach (var iw in mlp.Interwikis.Where(iw => iw.Code == langcode))
{
Console.WriteLine("Page {0}:{1} has WikiData item {2}", langcode, iw.Title, mlp.WikiDataItem);
}
}
// Excluding pages with WikiData item
if (!fullcheck)
{
mlpl = mlpl.Where(mlp => !mlp.IsOnWikiData).ToList();
}
GetLocalInterwiki(mlpl, langcode, projectcode);
foreach (var mlp in mlpl.Where(mlp => !mlp.IsOnWikiData && mlp.Interwikis.All(iw => iw.Code == langcode)))
{
foreach (var iw in mlp.Interwikis.Where(iw => iw.Code == langcode))
{
Console.WriteLine("Page {0}:{1} has no local links", langcode, iw.Title);
}
}
// Excluding pages without local interwikis
mlpl = mlpl.Where(mlp => mlp.Interwikis.Any(iw => iw.Code != langcode)).ToList();
if (mlpl.Count == 0)
{
_commonlog.LogData("Nothing to check", 2);
return;
}
_commonlog.LogData("Groups to check:", mlpl.Count.ToString(), 1);
for (int it = 1; it < IterationLimit; it++)
{
var langGroups = mlpl.Where(mlp => !mlp.HasConflict)
.SelectMany(mlp => mlp.Interwikis)
.Where(iw => !iw.IsRedirect && !iw.IsExcluded && !iw.IsToSection &&
(!iw.IsChecked || !iw.IsWdChecked || !iw.IsLocalIwChecked))
.GroupBy(iw => iw.Code);
if (!langGroups.Any())
{
break;
}
var primaryGroup = langGroups.OrderByDescending(g => g.Count()).First();
var primaryCode = primaryGroup.Key;
var primaryArray = primaryGroup.Select(iw => iw.Title).Distinct().ToArray();
logstring = string.Format("Iteration {0}. Check {1} links of language {2}...", it, primaryArray.Length, primaryCode);
_commonlog.LogData(logstring, 1);
GetProperTitles(mlpl, primaryCode, projectcode);
GetWikidataLinks(mlpl, primaryCode, projectcode);
GetLocalInterwiki(mlpl, primaryCode, projectcode);
CheckConflicts(mlpl);
CheckNamespaces(mlpl, primaryCode);
if (it >= 499)
{
foreach (var mlp in mlpl)
{
var uncheckedIw =
mlp.Interwikis.FirstOrDefault(iw => iw.Code == primaryCode && primaryArray.Contains(iw.Title));
if (uncheckedIw == null)
{
continue;
}
mlp.HasConflict = true;
mlp.ConflictDescription =
string.Format("Too much iterations. Started on {0}:{1}. Looped on {2}:{3}",
langcode,
mlp.Interwikis[0].Title,
primaryCode,
uncheckedIw.Title);
}
}
}
var mlpConflicts = mlpl.Where(mlp => mlp.HasConflict).ToList();
var mlpUpdates = mlpl.Where(mlp => !mlp.HasConflict && mlp.IsOnWikiData).ToList();
var mlpCreates = mlpl.Where(mlp => !mlp.HasConflict && !mlp.IsOnWikiData && mlp.Interwikis.Count > 1).ToList();
if (mlpCreates.Count > 0)
{
_commonlog.LogData("Proposed creations:", mlpCreates.Count.ToString(), 1);
if (onlyupdate)
{
_commonlog.LogData("Creation prohibited", 1);
}
}
if (mlpUpdates.Count > 0)
{
_commonlog.LogData("Proposed updates:", mlpUpdates.Count.ToString(), 1);
}
if (mlpConflicts.Count > 0)
{
_commonlog.LogData("Conflicts:", mlpConflicts.Count.ToString(), 1);
foreach (var mlp in mlpConflicts)
{
_conflictlog.LogData(mlp.ConflictDescription, 2);
}
}
foreach (var mlpCreate in mlpCreates)
{
if (projectcode != "wikipedia" || onlyupdate)
{
continue;
}
var iwList = mlpCreate
.Interwikis
.Where(iw => !iw.IsRedirect && !iw.IsExcluded && !iw.IsToSection)
.ToDictionary(iw => iw.Code, iw => iw.Title);
if (iwList.Count < _botConfiguration.MinInterwikiNumber)
{
Console.WriteLine("Page {0}:{1} has not sufficient number of interwikis. Skipping...",
iwList.ElementAt(0).Key, iwList.ElementAt(0).Value);
continue;
}
Item wdItem = new Item(_wikidataSite);
iwList = ReorderInterwiki(iwList);
var projectIwList = _wikiCodes.ToProjectCodes(iwList);
try
{
wdItem.createItem(projectIwList, MakeCreationSummary(iwList, projectcode));
logstring = MakeActionLogString(iwList, projectcode);
_actionlog.LogData(logstring, 3);
}
catch (Exception e)
{
if (iwList.ContainsKey(langcode))
{
logstring = string.Format("Problem when page {0} creation", iwList[langcode]);
_actionlog.LogData(logstring, 3);
}
_actionlog.LogData(e.ToString(), 3);
}
}
foreach (var mlpUpdate in mlpUpdates)
{
var properIws = mlpUpdate
.Interwikis
.Where(iw => !iw.IsRedirect && !iw.IsExcluded && !iw.IsToSection);
if (properIws.All(iw => iw.IsOnWd))
{
Console.WriteLine("Page {0}:{1} has not sufficient additional of interwikis. Skipping...",
properIws.ElementAt(0).Code, properIws.ElementAt(0).Title);
continue;
}
var iwList =
ReorderInterwiki(properIws
.ToDictionary(iw => iw.Code, iw => iw.Title));
Item wdItem = new Item(_wikidataSite, mlpUpdate.WikiDataItem);
wdItem.Load();
_wikiCodes.SetProjectCode(projectcode);
var oldIwList = _wikiCodes.ToLanguageCodes(wdItem.links);
var projectIwList = _wikiCodes.ToProjectCodes(iwList);
var addList = _wikiCodes.MakeAddList(oldIwList, iwList);
var updateList = _wikiCodes.MakeReplaceList(oldIwList, iwList);
try
{
wdItem.setSiteLink(projectIwList, MakeSummary(addList, updateList, projectcode, false));
logstring = MakeActionLogString(addList, updateList, projectcode, false);
_actionlog.LogData(logstring, 3);
}
catch (Exception e)
{
if (iwList.ContainsKey(langcode))
{
logstring = string.Format("Problem when link {0} adding", iwList[langcode]);
_actionlog.LogData(logstring, 3);
}
_actionlog.LogData(e.ToString(), 3);
}
}
}
private void ProcessPortion(string langcode, string projectcode, List<string> pages, bool fullcheck, bool onlyupdate)
{
var mlpl = new List<MultilingualPage>();
foreach (var page in pages)
{
var mlp = new MultilingualPage();
mlp.Interwikis.Add(new InterwikiItem { Title = page, Code = langcode });
mlpl.Add(mlp);
}
_commonlog.LogData("First page in the portion:", pages[0], 2);
var logstring = string.Format("Iteration {0}. Check {1} links of language {2}...", 0, pages.Count, langcode);
_commonlog.LogData(logstring, 1);
GetProperTitles(mlpl, langcode, projectcode);
GetWikidataLinks(mlpl, langcode, projectcode);
foreach (var mlp in mlpl.Where(mlp => mlp.IsOnWikiData))
{
foreach (var iw in mlp.Interwikis.Where(iw => iw.Code == langcode))
{
Console.WriteLine("Page {0}:{1} has WikiData item {2}", langcode, iw.Title, mlp.WikiDataItem);
}
}
// Excluding pages with WikiData item
if (!fullcheck)
mlpl = mlpl.Where(mlp => !mlp.IsOnWikiData).ToList();
GetLocalInterwiki(mlpl, langcode, projectcode);
foreach (var mlp in mlpl.Where(mlp => !mlp.IsOnWikiData && mlp.Interwikis.All(iw => iw.Code == langcode)))
{
foreach (var iw in mlp.Interwikis.Where(iw => iw.Code == langcode))
{
Console.WriteLine("Page {0}:{1} has no local links", langcode, iw.Title);
}
}
// Excluding pages without local interwikis
mlpl = mlpl.Where(mlp => mlp.Interwikis.Any(iw => iw.Code != langcode)).ToList();
if (mlpl.Count == 0)
{
_commonlog.LogData("Nothing to check", 2);
return;
}
_commonlog.LogData("Groups to check:", mlpl.Count.ToString(), 1);
for (int it = 1; it < IterationLimit; it++)
{
var langGroups = mlpl.Where(mlp => !mlp.HasConflict)
.SelectMany(mlp => mlp.Interwikis)
.Where(iw => !iw.IsRedirect && !iw.IsExcluded && !iw.IsToSection
&& (!iw.IsChecked || !iw.IsWdChecked || !iw.IsLocalIwChecked))
.GroupBy(iw => iw.Code);
if (!langGroups.Any())
break;
var primaryGroup = langGroups.OrderByDescending(g => g.Count()).First();
var primaryCode = primaryGroup.Key;
var primaryArray = primaryGroup.Select(iw => iw.Title).Distinct().ToArray();
logstring = string.Format("Iteration {0}. Check {1} links of language {2}...", it, primaryArray.Length, primaryCode);
_commonlog.LogData(logstring, 1);
GetProperTitles(mlpl, primaryCode, projectcode);
GetWikidataLinks(mlpl, primaryCode, projectcode);
GetLocalInterwiki(mlpl, primaryCode, projectcode);
CheckConflicts(mlpl);
CheckNamespaces(mlpl, primaryCode);
if (it >= 499)
{
foreach (var mlp in mlpl)
{
var uncheckedIw =
mlp.Interwikis.FirstOrDefault(iw => iw.Code == primaryCode && primaryArray.Contains(iw.Title));
if (uncheckedIw == null)
continue;
mlp.HasConflict = true;
mlp.ConflictDescription =
string.Format("Too much iterations. Started on {0}:{1}. Looped on {2}:{3}",
langcode,
mlp.Interwikis[0].Title,
primaryCode,
uncheckedIw.Title);
}
}
}
var mlpConflicts = mlpl.Where(mlp => mlp.HasConflict).ToList();
var mlpUpdates = mlpl.Where(mlp => !mlp.HasConflict && mlp.IsOnWikiData).ToList();
var mlpCreates = mlpl.Where(mlp => !mlp.HasConflict && !mlp.IsOnWikiData && mlp.Interwikis.Count > 1).ToList();
if (mlpCreates.Count > 0)
{
_commonlog.LogData("Proposed creations:", mlpCreates.Count.ToString(), 1);
if (onlyupdate)
_commonlog.LogData("Creation prohibited", 1);
}
if (mlpUpdates.Count > 0)
{
_commonlog.LogData("Proposed updates:", mlpUpdates.Count.ToString(), 1);
}
if (mlpConflicts.Count > 0)
{
_commonlog.LogData("Conflicts:", mlpConflicts.Count.ToString(), 1);
foreach (var mlp in mlpConflicts)
{
_conflictlog.LogData(mlp.ConflictDescription, 2);
}
}
foreach (var mlpCreate in mlpCreates)
{
if (projectcode != "wikipedia" || onlyupdate)
continue;
var iwList = mlpCreate
.Interwikis
.Where(iw => !iw.IsRedirect && !iw.IsExcluded && !iw.IsToSection)
.ToDictionary(iw => iw.Code, iw => iw.Title);
if (iwList.Count < _botConfiguration.MinInterwikiNumber)
{
Console.WriteLine("Page {0}:{1} has not sufficient number of interwikis. Skipping...",
iwList.ElementAt(0).Key, iwList.ElementAt(0).Value);
continue;
}
Item wdItem = new Item(_wikidataSite);
iwList = ReorderInterwiki(iwList);
var projectIwList = _wikiCodes.ToProjectCodes(iwList);
try
{
wdItem.createItem(projectIwList, MakeCreationSummary(iwList, projectcode));
logstring = MakeActionLogString(iwList, projectcode);
_actionlog.LogData(logstring, 3);
}
catch (Exception e)
{
if (iwList.ContainsKey(langcode))
{
logstring = string.Format("Problem when page {0} creation", iwList[langcode]);
_actionlog.LogData(logstring, 3);
}
_actionlog.LogData(e.ToString(), 3);
}
}
foreach (var mlpUpdate in mlpUpdates)
{
var properIws = mlpUpdate
.Interwikis
.Where(iw => !iw.IsRedirect && !iw.IsExcluded && !iw.IsToSection);
if (properIws.All(iw => iw.IsOnWd))
{
Console.WriteLine("Page {0}:{1} has not sufficient additional of interwikis. Skipping...",
properIws.ElementAt(0).Code, properIws.ElementAt(0).Title);
continue;
}
var iwList =
ReorderInterwiki(properIws
.ToDictionary(iw => iw.Code, iw => iw.Title));
Item wdItem = new Item(_wikidataSite, mlpUpdate.WikiDataItem);
wdItem.Load();
_wikiCodes.SetProjectCode(projectcode);
var oldIwList = _wikiCodes.ToLanguageCodes(wdItem.links);
var projectIwList = _wikiCodes.ToProjectCodes(iwList);
var addList = _wikiCodes.MakeAddList(oldIwList, iwList);
var updateList = _wikiCodes.MakeReplaceList(oldIwList, iwList);
try
{
wdItem.setSiteLink(projectIwList, MakeSummary(addList, updateList, projectcode, false));
logstring = MakeActionLogString(addList, updateList, projectcode, false);
_actionlog.LogData(logstring, 3);
}
catch (Exception e)
{
if (iwList.ContainsKey(langcode))
{
logstring = string.Format("Problem when link {0} adding", iwList[langcode]);
_actionlog.LogData(logstring, 3);
}
_actionlog.LogData(e.ToString(), 3);
}
}
}
