public async Task WpBackLinksGeneratorTest()
{
var site = await WpTest2SiteAsync;
var blg = new BacklinksGenerator(site, "Albert Einstein")
{
PaginationSize = 100
};
var pages = await blg.EnumPagesAsync().Take(100).ToList();
ShallowTrace(pages, 1);
Assert.Contains(pages, p => p.Title == "Judaism");
Assert.Contains(pages, p => p.Title == "Physics");
Assert.Contains(pages, p => p.Title == "United States");
}
static async Task Main(string[] args)
{
using var client = new WikiClient();
var site = new WikiSite(client, "https://crystalpool.cxuesong.com/api.php");
await site.Initialization;
await site.LoginAsync("", "");
Console.WriteLine("Logged in to {0} as {1}.", site, site.AccountInfo);
var gen = new BacklinksGenerator(site, "Property:P97")
{
PaginationSize = 50,
NamespaceIds = new[] { 120 }
};
await foreach (var batch in gen.EnumItemsAsync().Select(stub => new Entity(site, stub.Title.Split(":")[1])).Buffer(50))
{
await batch.RefreshAsync(EntityQueryOptions.FetchClaims | EntityQueryOptions.FetchLabels, new[] { "en" });
foreach (var item in batch)
{
Console.WriteLine("Processing {0}.", item);
var mannerClaim = item.Claims["P97"].SingleOrDefault();
if (mannerClaim == null)
{
continue;
}
var timeClaim = item.Claims["P96"].SingleOrDefault();
if (timeClaim == null)
{
Console.WriteLine("No death time claim for {0}.", item);
continue;
}
timeClaim.Qualifiers.Add(new Snak("P97", mannerClaim.MainSnak.RawDataValue, BuiltInDataTypes.WikibaseItem));
foreach (var q in mannerClaim.Qualifiers)
{
timeClaim.Qualifiers.Add(q);
}
await item.EditAsync(new[] {
new EntityEditEntry(nameof(item.Claims), timeClaim),
new EntityEditEntry(nameof(item.Claims), mannerClaim, EntityEditEntryState.Removed)
}, "Bot: Moving P97 as a qualifier of P96.", EntityEditOptions.Bot);
}
}
await site.LogoutAsync();
}
static async Task Main(string[] args)
{
using var client = new WikiClient();
var site = new WikiSite(client, "https://crystalpool.cxuesong.com/api.php");
await site.Initialization;
site.ModificationThrottler.ThrottleTime = TimeSpan.FromSeconds(1);
await site.LoginAsync("", "");
Console.WriteLine("Logged in to {0} as {1}.", site, site.AccountInfo);
var gen = new BacklinksGenerator(site, "Item:Q171")
{
PaginationSize = 50,
NamespaceIds = new[] { 120 }
};
const string replaceFrom = "族群情仇";
const string replaceTo = "雷影交加";
await foreach (var batch in gen.EnumItemsAsync().Select(stub => new Entity(site, stub.Title.Split(":")[1])).Buffer(50))
{
await batch.RefreshAsync(EntityQueryOptions.FetchLabels | EntityQueryOptions.FetchDescriptions | EntityQueryOptions.FetchAliases,
new[] { "en", "zh", "zh-cn", "zh-tw" });
foreach (var item in batch)
{
Console.WriteLine("Processing {0}.", item);
var edits = new List <EntityEditEntry>();
foreach (var label in item.Labels)
{
if (label.Text.Contains(replaceFrom))
{
edits.Add(new EntityEditEntry(nameof(item.Labels), new WbMonolingualText(label.Language, label.Text.Replace(replaceFrom, replaceTo))));
}
}
foreach (var desc in item.Descriptions)
{
if (desc.Text.Contains(replaceFrom))
{
edits.Add(new EntityEditEntry(nameof(item.Descriptions), new WbMonolingualText(desc.Language, desc.Text.Replace(replaceFrom, replaceTo))));
}
}
foreach (var alias in item.Aliases)
{
if (alias.Text.Contains(replaceFrom))
{
edits.Add(new EntityEditEntry(nameof(item.Aliases), new WbMonolingualText(alias.Language, alias.Text.Replace(replaceFrom, replaceTo))));
}
}
if (edits.Count > 0)
{
Console.WriteLine("Pending edits:");
foreach (var edit in edits)
{
Console.WriteLine(" {0} {1} {2}", edit.PropertyName, edit.State, edit.Value);
}
await item.EditAsync(edits, $"Bot: Text replace: {replaceFrom} -> {replaceTo}.", EntityEditOptions.Bot);
}
}
}
await site.LogoutAsync();
}
private static async Task UpdateLabels(WikiSite site, string originEntity)
{
var apg = new BacklinksGenerator(site, originEntity)
{
NamespaceIds = new[] { site.Namespaces["Item"].Id },
RedirectsFilter = PropertyFilterOption.WithoutProperty,
PaginationSize = 100
};
var itemCounter = 0;
await foreach (var stubs in apg.EnumItemsAsync().Buffer(100))
{
var items = stubs.Select(s => new Entity(site, WikiLink.Parse(site, s.Title).Title)).ToList();
await items.RefreshAsync(EntityQueryOptions.FetchLabels | EntityQueryOptions.FetchAliases | EntityQueryOptions.FetchSiteLinks);
foreach (var item in items)
{
var enLabels = new HashSet <string>();
{
var enSiteLink = item.SiteLinks.FirstOrDefault(sl => sl.Site == "enwarriorswiki")?.Title;
if (!string.IsNullOrEmpty(enSiteLink))
{
enLabels.Add(LabelFromTitle(enSiteLink));
}
enLabels.Add(item.Labels["en"]);
enLabels.Add(item.Labels["en-us"]);
enLabels.Add(item.Labels["en-gb"]);
foreach (var alias in item.Aliases["en"])
{
enLabels.Add(alias);
}
enLabels.Remove(null);
}
foreach (var siteLink in item.SiteLinks.Where(sl => sl.Site.EndsWith("warriorswiki")))
{
var language = siteLink.Site.Substring(0, siteLink.Site.Length - 12);
if (language == "zh")
{
continue;
}
var culture = CultureInfo.GetCultureInfo(language);
var label = Regex.Replace(siteLink.Title, @"(?!<=^)\s*\(.+", "");
var curLabel = item.Labels[language];
//if (language != "en")
//{
// if (enLabels.Contains(curLabel))
// {
// Console.Write("{0}: Remove label: {1}@{2}", item, curLabel, language);
// await item.EditAsync(new[]
// {
// new EntityEditEntry(nameof(item.Labels), new WbMonolingualText(language, "dummy"), EntityEditEntryState.Removed)
// }, "", EntityEditOptions.Bot);
// Console.WriteLine();
// }
// foreach (var alias in item.Aliases[language])
// {
// if (enLabels.Contains(alias))
// {
// Console.Write("{0}: Remove alias: {1}@{2}", item, alias, language);
// await item.EditAsync(new[]
// {
// new EntityEditEntry(nameof(item.Aliases), new WbMonolingualText(language, alias), EntityEditEntryState.Removed)
// }, "", EntityEditOptions.Bot);
// Console.WriteLine();
// }
// }
//}
if (!enLabels.Contains(label) && (curLabel == null || label.ToLower(culture) != curLabel.ToLower(culture)))
{
var convertToAlias = !string.IsNullOrWhiteSpace(curLabel) && !enLabels.Contains(curLabel);
Console.Write("{0}: {1}: {2} -> {3}", item, language, curLabel, label);
var changes = new List <EntityEditEntry>
{
new EntityEditEntry(nameof(item.Labels), new WbMonolingualText(language, label))
};
if (convertToAlias)
{
Console.Write(" (New alias)");
changes.Add(new EntityEditEntry(nameof(item.Aliases), new WbMonolingualText(language, curLabel)));
}
await item.EditAsync(changes, "Update label from " + siteLink.Site + ".", EntityEditOptions.Bot);
Console.WriteLine();
}
}
}
itemCounter += items.Count;
Console.WriteLine("Processed {0} items.", itemCounter);
}
}
private static async Task FetchLabels(WikiSite site, Func <string, Task <WikiSite> > getExternalSiteAsync, string originEntity)
{
var apg = new BacklinksGenerator(site, originEntity)
{
NamespaceIds = new[] { site.Namespaces["Item"].Id },
RedirectsFilter = PropertyFilterOption.WithoutProperty,
PaginationSize = 100
};
var itemCounter = 0;
var enSite = await getExternalSiteAsync("en");
await foreach (var stubs in apg.EnumItemsAsync().Buffer(100))
{
var items = stubs.Select(s => new Entity(site, WikiLink.Parse(site, s.Title).Title)).ToList();
await items.RefreshAsync(EntityQueryOptions.FetchLabels | EntityQueryOptions.FetchSiteLinks, new List <string> {
"en"
});
var enwwPages = items.ToDictionary(i => i, i =>
{
var enwwtitle = i.SiteLinks.ContainsKey("enwarriorswiki") ? i.SiteLinks["enwarriorswiki"].Title : null;
if (string.IsNullOrEmpty(enwwtitle))
{
Console.WriteLine("{0}: No enww sitelink available.", i);
return(null);
}
return(new WikiPage(enSite, enwwtitle));
});
Console.WriteLine("Fetching language links from enww.");
await enwwPages.Values.Where(p => p != null).RefreshAsync(new WikiPageQueryProvider {
Properties = { new LanguageLinksPropertyProvider() }
});
foreach (var item in items)
{
var enPage = enwwPages[item];
if (enPage == null)
{
continue;
}
foreach (var langLink in enPage.GetPropertyGroup <LanguageLinksPropertyGroup>().LanguageLinks
// Wikia returns duplicate language links
.Select(l => (l.Language, l.Title)).Distinct())
{
if (langLink.Language == "zh")
{
continue;
}
var siteName = langLink.Language + "warriorswiki";
if (item.SiteLinks.ContainsKey(siteName))
{
continue;
}
Console.Write("{0}: Add {1}:{2}...", item, siteName, langLink.Title);
try
{
await item.EditAsync(new[]
{
new EntityEditEntry(nameof(item.SiteLinks),
new EntitySiteLink(siteName, langLink.Title))
},
"Add sitelink from enwarriorswiki.", EntityEditOptions.Bot);
Console.WriteLine("Success.");
}
catch (OperationFailedException ex) when(ex.ErrorCode == "no-external-page")
{
Console.WriteLine("No external page.");
}
catch (OperationFailedException ex) when(ex.ErrorCode == "failed-save")
{
Console.WriteLine("Save failed.");
Console.WriteLine(ex.ErrorMessage);
}
}
}
itemCounter += items.Count;
Console.WriteLine("Processed {0} items.", itemCounter);
}
}