/// <summary>
/// Construct a sequence of <see cref="WikiPageStub"/> from the given page IDs.
/// </summary>
/// <param name="site">The site in which to query for the pages.</param>
/// <param name="ids">The page IDs to query.</param>
/// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="ids"/> is <c>null</c>.</exception>
/// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns>
/// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks>
public static IAsyncEnumerable <WikiPageStub> FromPageIds(WikiSite site, IEnumerable <int> ids)
{
return(AsyncEnumerableFactory.FromAsyncGenerator <WikiPageStub>(async(sink, ct) =>
{
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
? 500
: 50;
foreach (var partition in ids.Partition(titleLimit))
{
var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "query",
pageids = MediaWikiHelper.JoinValues(partition),
}), ct);
Debug.Assert(jresult["query"] != null);
var jpages = jresult["query"]["pages"];
foreach (var id in partition)
{
var jpage = jpages[id.ToString(CultureInfo.InvariantCulture)];
if (jpage["missing"] == null)
{
sink.Yield(new WikiPageStub(id, (string)jpage["title"], (int)jpage["ns"]));
}
else
{
sink.Yield(new WikiPageStub(id, MissingPageTitle, UnknownNamespaceId));
}
}
await sink.Wait();
}
}));
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object> > EnumParameters(MediaWikiVersion version)
{
return(new OrderedKeyValuePairs <string, object>
{
{ "ppprop", SelectedProperties == null ? null : MediaWikiHelper.JoinValues(SelectedProperties) }
});
}
private IEnumerable <KeyValuePair <string, object> > EnumParams(bool isList)
{
var prefix = isList ? null : "g";
var dict = new Dictionary <string, object>
{
{ prefix + "rcdir", TimeAscending ? "newer" : "older" },
{ prefix + "rcstart", StartTime },
{ prefix + "rcend", EndTime },
{ prefix + "rcnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) },
{ prefix + "rcuser", UserName },
{ prefix + "rcexcludeuser", ExcludedUserName },
{ prefix + "rctag", Tag },
{ prefix + "rctype", ParseRecentChangesTypes(TypeFilters) },
{ prefix + "rcshow", ParseFilters() },
{ prefix + "rctoponly", LastRevisionsOnly },
{ prefix + "rclimit", PaginationSize }
};
if (isList)
{
var fields = "user|userid|comment|parsedcomment|flags|timestamp|title|ids|sizes|redirect|loginfo|tags|sha1";
if (Site.AccountInfo.HasRight(UserRights.Patrol))
{
fields += "|patrolled";
}
dict.Add("rcprop", fields);
}
return(dict);
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object> > EnumListParameters()
{
if ((TargetTitle != null) == (TargetPageId != null))
{
throw new ArgumentException(string.Format(Prompts.ExceptionArgumentExpectEitherNull2, nameof(TargetTitle), nameof(TargetPageId)));
}
var actualPaginationSize = PaginationSize;
if (AllowRedirectedLinks)
{
// When the blredirect parameter is set, this module behaves slightly differently.
// bllimit applies to both levels separately: if e.g. bllimit=10,
// at most 10 first-level pages (pages that link to bltitle) and
// 10 second-level pages (pages that link to bltitle through a redirect) will be listed.
// Continuing queries also works differently.
actualPaginationSize = Math.Max(1, PaginationSize / 2);
}
return(new Dictionary <string, object>
{
{ "bltitle", TargetTitle },
{ "blpageid", TargetPageId },
{ "blnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) },
{ "blfilterredir", RedirectsFilter.ToString("redirects", "nonredirects") },
{ "bllimit", actualPaginationSize },
{ "blredirect", AllowRedirectedLinks }
});
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object> > EnumListParameters()
{
var dict = new Dictionary <string, object>
{
{ "srsearch", Keyword },
{ "srnamespace", NamespaceIds == null ? "*" : MediaWikiHelper.JoinValues(NamespaceIds) },
{ "srwhat", MatchingField },
{ "srlimit", PaginationSize },
{ "srinterwiki", IncludesInterwiki },
{ "srbackend", BackendName }
};
// Include redirect pages in the search. From 1.23 onwards, redirects are always included. (Removed in 1.23)
if (Site.SiteInfo.Version < new MediaWikiVersion(1, 23))
{
dict["srredirects"] = true;
}
dict["srwhat"] = MatchingField switch
{
SearchableField.Title => "title",
SearchableField.Text => "text",
SearchableField.NearMatch => "nearmatch",
_ => throw new ArgumentOutOfRangeException()
};
return(dict);
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object?> > EnumListParameters()
{
return(new Dictionary <string, object>
{
{ "tlnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) }, { "tllimit", PaginationSize }, { "tltemplates", MatchingTitles == null ? null : MediaWikiHelper.JoinValues(MatchingTitles) },
{ "tldir", OrderDescending ? "descending" : "ascending" }
});
}
public async Task Issue67()
{
var site = await WpEnSiteAsync;
var items = await new CategoriesGenerator(site)
{
PageTitle = MediaWikiHelper.JoinValues(new[] { "Test", ".test", "Test_(Unix)", "Test_(assessment)" }),
}.EnumItemsAsync().ToListAsync();
ShallowTrace(items);
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object> > EnumListParameters()
{
return(new Dictionary <string, object>
{
{ "eititle", TargetTitle },
{ "einamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) },
{ "eifilterredir", RedirectsFilter.ToString("redirects", "nonredirects") },
{ "eilimit", PaginationSize }
});
}
/// <summary>
/// Refresh a sequence of revisions by revid, along with their owner pages.
/// </summary>
/// <remarks>
/// <para>If there's invalid revision id in <paramref name="revIds"/>, an <see cref="ArgumentException"/> will be thrown while enumerating.</para>
/// </remarks>
public static IAsyncEnumerable <Revision> FetchRevisionsAsync(WikiSite site, IEnumerable <int> revIds, IWikiPageQueryProvider options, CancellationToken cancellationToken)
{
if (revIds == null)
{
throw new ArgumentNullException(nameof(revIds));
}
var queryParams = options.EnumParameters(site.SiteInfo.Version).ToDictionary();
// Remove any rvlimit magic word generated by RevisionsPropertyProvider.
// We are only fetching by revisions.
queryParams.Remove("rvlimit");
var titleLimit = options.GetMaxPaginationSize(site.SiteInfo.Version, site.AccountInfo.HasRight(UserRights.ApiHighLimits));
return(AsyncEnumerableFactory.FromAsyncGenerator <Revision>(async sink =>
{
// Page ID --> Page Stub
var stubDict = new Dictionary <int, WikiPageStub>();
var revDict = new Dictionary <int, Revision>();
using (site.BeginActionScope(null, (object)revIds))
{
foreach (var partition in revIds.Partition(titleLimit))
{
site.Logger.LogDebug("Fetching {Count} revisions from {Site}.", partition.Count, site);
queryParams["revids"] = MediaWikiHelper.JoinValues(partition);
var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken);
var jpages = (JObject)jobj["query"]["pages"];
// Generate stubs first
foreach (var p in jpages)
{
var jrevs = p.Value["revisions"];
if (jrevs == null || !jrevs.HasValues)
{
continue;
}
var id = Convert.ToInt32(p.Key);
if (!stubDict.TryGetValue(id, out var stub))
{
stub = new WikiPageStub(id, (string)p.Value["title"], (int)p.Value["ns"]);
stubDict.Add(id, stub);
}
foreach (var jrev in jrevs)
{
var rev = jrev.ToObject <Revision>(Utility.WikiJsonSerializer);
rev.Page = stub;
revDict.Add(rev.Id, rev);
}
}
await sink.YieldAndWait(partition.Select(id => revDict.TryGetValue(id, out var rev) ? rev : null));
}
}
}));
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object> > EnumParameters(MediaWikiVersion version)
{
var p = new OrderedKeyValuePairs <string, object>
{
{ "lhprop", "pageid|title|redirect" },
{ "lhshow", RedirectFilter.ToString("redirect", "!redirect", null) }
};
if (NamespaceSelection != null)
{
p.Add("lhnamespace", MediaWikiHelper.JoinValues(NamespaceSelection));
}
return(p);
}
/// <inheritdoc/>
public override IEnumerable <KeyValuePair <string, object> > EnumListParameters()
{
if (string.IsNullOrEmpty(CategoryTitle))
{
throw new InvalidOperationException(string.Format(Prompts.ExceptionArgumentIsEmpty1, nameof(CategoryTitle)));
}
return(new Dictionary <string, object>
{
{ "cmtitle", CategoryTitle },
{ "cmlimit", PaginationSize },
{ "cmnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) },
{ "cmtype", ParseMemberTypes(MemberTypes) }
});
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object> > EnumParameters(MediaWikiVersion version)
{
var p = new OrderedKeyValuePairs <string, object>
{
{ "clprop", "sortkey|timestamp|hidden" },
{ "clshow", HiddenCategoryFilter.ToString("hidden", "!hidden", null) }
};
if (CategorySelection != null)
{
p.Add("clcategories", MediaWikiHelper.JoinValues(CategorySelection));
}
return(p);
}
public static async Task RefreshEntitiesAsync(IEnumerable <Entity> entities, EntityQueryOptions options,
IEnumerable <string> languages, CancellationToken cancellationToken)
{
if (entities == null)
{
throw new ArgumentNullException(nameof(entities));
}
var langs = languages == null ? null : MediaWikiHelper.JoinValues(languages);
if (string.IsNullOrEmpty(langs))
{
langs = null;
}
// You can even fetch pages from different sites.
foreach (var siteEntities in entities.GroupBy(p => p.Site))
{
var site = siteEntities.Key;
var req = BuildQueryOptions(langs, options);
req["action"] = "wbgetentities";
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50;
using (site.BeginActionScope(entities, options))
{
foreach (var partition in siteEntities.Partition(titleLimit).Select(partition => partition.ToList()))
{
//site.Logger.LogDebug("Fetching {Count} pages from {Site}.", partition.Count, site);
// We use ids to query pages.
req["ids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id));
var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(req), cancellationToken);
var jentities = (JObject)jresult["entities"];
foreach (var entity in partition)
{
var jentity = jentities[entity.Id];
// We can write Q123456 as q123456 in query params, but server will return Q123456 anyway.
if (jentity == null)
{
jentity = jentities.Properties().FirstOrDefault(p =>
string.Equals(p.Name, entity.Id, StringComparison.OrdinalIgnoreCase));
}
if (jentity == null)
{
throw new UnexpectedDataException($"Cannot find the entity with id {entity.Id} in the response.");
}
entity.LoadFromJson(jentity, options, false);
}
}
}
}
}
/// <summary>
/// Construct a sequence of <see cref="WikiPageStub"/> from the given page titles.
/// </summary>
/// <param name="site">The site in which to query for the pages.</param>
/// <param name="titles">The page IDs to query.</param>
/// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="titles"/> is <c>null</c>.</exception>
/// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns>
/// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks>
public static IAsyncEnumerable <WikiPageStub> FromPageTitles(WikiSite site, IEnumerable <string> titles)
{
if (site == null)
{
throw new ArgumentNullException(nameof(site));
}
if (titles == null)
{
throw new ArgumentNullException(nameof(titles));
}
return(AsyncEnumerableFactory.FromAsyncGenerator <WikiPageStub>(async(sink, ct) =>
{
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
? 500
: 50;
foreach (var partition in titles.Partition(titleLimit))
{
var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "query", titles = MediaWikiHelper.JoinValues(partition),
}), ct);
Debug.Assert(jresult["query"] != null);
// Process title normalization.
var normalizedDict = jresult["query"]["normalized"]?.ToDictionary(n => (string)n["from"],
n => (string)n["to"]);
var pageDict = ((JObject)jresult["query"]["pages"]).Properties()
.ToDictionary(p => (string)p.Value["title"], p => p.Value);
foreach (var name in partition)
{
if (normalizedDict == null || !normalizedDict.TryGetValue(name, out var normalizedName))
{
normalizedName = name;
}
var jpage = pageDict[normalizedName];
if (jpage["missing"] == null)
{
sink.Yield(new WikiPageStub((int)jpage["pageid"], (string)jpage["title"], (int)jpage["ns"]));
}
else
{
sink.Yield(new WikiPageStub(MissingPageIdMask, (string)jpage["title"], (int)jpage["ns"]));
}
}
await sink.Wait();
}
}));
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object> > EnumParameters(MediaWikiVersion version)
{
var p = new OrderedKeyValuePairs <string, object>
{
{
"rvprop", FetchContent
? "ids|timestamp|flags|comment|user|userid|contentmodel|sha1|tags|size|content"
: "ids|timestamp|flags|comment|user|userid|contentmodel|sha1|tags|size"
}
};
if (Slots != null || version >= new MediaWikiVersion(1, 32))
{
// If user specified Slots explicitly, then we will respect it regardless of MW version.
p.Add("rvslots", Slots == null ? RevisionSlot.MainSlotName : MediaWikiHelper.JoinValues(Slots));
}
return(p);
}
/// <inheritdoc/>
public override IEnumerable <KeyValuePair <string, object> > EnumListParameters()
{
var dict = new Dictionary <string, object>
{
{ "rnnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) },
{ "rnlimit", PaginationSize },
};
if (Site.SiteInfo.Version >= new Version(1, 26))
{
dict.Add("rnfilterredir", RedirectsFilter.ToString("redirects", "nonredirects"));
}
else if (RedirectsFilter == PropertyFilterOption.WithProperty)
{
dict.Add("rnredirect", true);
// for MW 1.26-, we cannot really implement RedirectsFilter == PropertyFilterOption.WithoutProperty
}
return(dict);
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object> > EnumListParameters()
{
var prop = new Dictionary <string, object>
{
{ "gsradius", Radius },
{ "gsnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) },
{ "gsprimary", IncludesSecondaryCoordinates ? "all" : "primary" },
{ "gsglobe", TargetCoordinate.Globe },
{ "gslimit", PaginationSize },
};
if (TargetTitle != null)
{
// When searching by page title, it would be better for MW API to
// assume `gsglobe` corresponds to the `globe` of that page,
// but there is currently no such behavior.
prop["gspage"] = TargetTitle;
}
else if (!BoundingRectangle.IsEmpty)
{
var rect = BoundingRectangle;
// This is way too large than the width acceptable by MW API.
// And this causes weird `Right` value.
if (rect.Width > 180)
{
throw new ArgumentException("Bounding box is too big.", nameof(BoundingRectangle));
}
rect.Normalize();
var right = rect.Right;
if (right > 180)
{
right -= 360;
}
prop["gsbbox"] = string.Format(CultureInfo.InvariantCulture, "{0}|{1}|{2}|{3}",
rect.Top, rect.Left, rect.Bottom, right);
}
else
{
prop["gscoord"] = TargetCoordinate.Latitude.ToString(CultureInfo.InvariantCulture)
+ "|" + TargetCoordinate.Longitude.ToString(CultureInfo.InvariantCulture);
}
return(prop);
}
public static IAsyncEnumerable <string> EntityIdsFromSiteLinksAsync(WikiSite site,
string siteName, IEnumerable <string> siteLinks)
{
Debug.Assert(siteName != null);
Debug.Assert(siteLinks != null);
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50;
return(AsyncEnumerableFactory.FromAsyncGenerator <string>(async(sink, ct) =>
{
var req = new OrderedKeyValuePairs <string, string>
{
{ "action", "wbgetentities" },
{ "props", "sitelinks" },
{ "sites", siteName },
{ "sitefilter", siteName },
};
using (site.BeginActionScope(siteLinks))
{
foreach (var partition in siteLinks.Partition(titleLimit).Select(partition => partition.ToList()))
{
//site.Logger.LogDebug("Fetching {Count} pages from {Site}.", partition.Count, site);
for (int i = 0; i < partition.Count; i++)
{
if (partition[i] == null)
{
throw new ArgumentException("Link titles contain null element.", nameof(siteLinks));
}
// Do some basic title normalization locally.
// Note Wikibase cannot even normalize the first letter case of the title.
partition[i] = partition[i].Trim(whitespaceAndUnderscore).Replace('_', ' ');
}
req["titles"] = MediaWikiHelper.JoinValues(partition);
var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(req), ct);
var jentities = (JObject)jresult["entities"];
var nameIdDict = jentities.PropertyValues().Where(e => e["missing"] == null)
.ToDictionary(e => (string)e["sitelinks"][siteName]["title"], e => (string)e["id"]);
await sink.YieldAndWait(partition.Select(title =>
nameIdDict.TryGetValue(title, out var id) ? id : null));
}
}
}));
}
/// <inheritdoc />
public override IEnumerable <KeyValuePair <string, object> > EnumListParameters()
{
var prop = new Dictionary <string, object>
{
{ "gsradius", Radius },
{ "gsnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) },
{ "gsprimary", IncludesSecondaryCoordinates ? "all" : "primary" },
{ "gsglobe", TargetCoordinate.Globe },
{ "gslimit", PaginationSize },
};
if (TargetTitle != null)
{
// When searching by page title, it would be better for MW API to
// assume `gsglobe` corresponds to the `globe` of that page,
// but there is currently no such behavior.
prop["gspage"] = TargetTitle;
}
else
{
prop["gscoord"] = TargetCoordinate.Latitude + "|" + TargetCoordinate.Longitude;
}
return(prop);
}
/// <inheritdoc/>
public override string ToString()
{
var tags = Tags == null ? null : MediaWikiHelper.JoinValues(Tags);
return($"Revision#{Id}, {Flags}, {tags}, SHA1={Sha1}");
}
/// <summary>
/// Refresh a sequence of pages.
/// </summary>
public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, IWikiPageQueryProvider options, CancellationToken cancellationToken)
{
if (pages == null)
{
throw new ArgumentNullException(nameof(pages));
}
// You can even fetch pages from different sites.
foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p)))
{
var site = sitePages.Key.Site;
var queryParams = options.EnumParameters().ToDictionary();
var titleLimit = options.GetMaxPaginationSize(site.AccountInfo.HasRight(UserRights.ApiHighLimits));
using (site.BeginActionScope(sitePages, options))
{
foreach (var partition in sitePages.Partition(titleLimit))
{
if (sitePages.Key.HasTitle)
{
// If a page has both title and ID information,
// we will use title anyway.
site.Logger.LogDebug("Fetching {Count} pages by title.", partition.Count);
queryParams["titles"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Title));
}
else
{
site.Logger.LogDebug("Fetching {Count} pages by ID.", partition.Count);
Debug.Assert(sitePages.All(p => p.PageStub.HasId));
queryParams["pageids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id));
}
// For single-page fetching, force fetching 1 revision only.
if (partition.Count == 1)
{
queryParams["rvlimit"] = 1;
}
else
{
queryParams.Remove("rvlimit");
}
var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken);
if (sitePages.Key.HasTitle)
{
// Process title normalization.
var normalized = jobj["query"]["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]);
// Process redirects.
var redirects = jobj["query"]["redirects"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]);
var pageInfoDict = ((JObject)jobj["query"]["pages"]).Properties()
.ToDictionary(p => (string)p.Value["title"]);
foreach (var page in partition)
{
var title = page.Title;
// Normalize the title first.
if (normalized?.ContainsKey(title) ?? false)
{
title = normalized[title];
}
// Then process the redirects.
var redirectTrace = new List <string>();
while (redirects?.ContainsKey(title) ?? false)
{
redirectTrace.Add(title); // Adds the last title
var next = redirects[title];
if (redirectTrace.Contains(next))
{
throw new InvalidOperationException($"Cannot resolve circular redirect: {string.Join("->", redirectTrace)}.");
}
title = next;
}
// Finally, get the page.
var pageInfo = pageInfoDict[title];
if (redirectTrace.Count > 0)
{
page.RedirectPath = redirectTrace;
}
MediaWikiHelper.PopulatePageFromJson(page, (JObject)pageInfo.Value, options);
}
}
else
{
foreach (var page in partition)
{
var jPage = (JObject)jobj["query"]["pages"][page.Id.ToString()];
MediaWikiHelper.PopulatePageFromJson(page, jPage, options);
}
}
}
}
}
}
private async Task ProgressiveEditAsync(IEnumerable <EntityEditEntry> edits, string summary, bool isBot,
bool strict, CancellationToken cancellationToken)
{
Debug.Assert(edits != null);
var checkbaseRev = true;
foreach (var prop in edits.GroupBy(e => e.PropertyName))
{
if (prop.Any(p => p.Value == null))
{
throw new ArgumentException($"Detected null value in {prop} entries.", nameof(edits));
}
switch (prop.Key)
{
case nameof(DataType):
throw new NotSupportedException("Setting data type is not possible in progressive mode.");
case nameof(Labels):
foreach (var p in prop)
{
var value = (WbMonolingualText)p.Value;
var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "wbsetlabel",
token = WikiSiteToken.Edit,
id = Id,
@new = Id == null ? FormatEntityType(Type) : null,
baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null,
bot = isBot,
summary = summary,
language = value.Language,
value = p.State == EntityEditEntryState.Updated ? value.Text : null,
}), cancellationToken);
LoadEntityMinimal(jresult["entity"]);
if (!strict)
{
checkbaseRev = false;
}
}
break;
case nameof(Descriptions):
foreach (var p in prop)
{
var value = (WbMonolingualText)p.Value;
var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "wbsetdescription",
token = WikiSiteToken.Edit,
id = Id,
@new = Id == null ? FormatEntityType(Type) : null,
baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null,
bot = isBot,
summary = summary,
language = value.Language,
value = p.State == EntityEditEntryState.Updated ? value.Text : null,
}), cancellationToken);
LoadEntityMinimal(jresult["entity"]);
if (!strict)
{
checkbaseRev = false;
}
}
break;
case nameof(Aliases):
{
var entries = prop.GroupBy(t => ((WbMonolingualText)t.Value).Language);
foreach (var langGroup in entries)
{
var addExpr = MediaWikiHelper.JoinValues(langGroup
.Where(e => e.State == EntityEditEntryState.Updated)
.Select(e => ((WbMonolingualText)e.Value).Text));
var removeExpr = MediaWikiHelper.JoinValues(langGroup
.Where(e => e.State == EntityEditEntryState.Removed)
.Select(e => ((WbMonolingualText)e.Value).Text));
var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "wbsetaliases",
token = WikiSiteToken.Edit,
id = Id,
@new = Id == null ? FormatEntityType(Type) : null,
baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null,
bot = isBot,
summary = summary,
language = langGroup.Key,
add = addExpr.Length == 0 ? null : addExpr,
remove = removeExpr.Length == 0 ? null : removeExpr,
}), cancellationToken);
LoadEntityMinimal(jresult["entity"]);
if (!strict)
{
checkbaseRev = false;
}
}
break;
}
case nameof(SiteLinks):
{
var entries = prop.GroupBy(t => ((EntitySiteLink)t.Value).Site);
foreach (var siteGroup in entries)
{
string link = null, badges = null;
try
{
var item = siteGroup.Single();
if (item.State == EntityEditEntryState.Updated)
{
var value = (EntitySiteLink)item.Value;
link = value.Title;
badges = MediaWikiHelper.JoinValues(value.Badges);
}
}
catch (InvalidOperationException)
{
throw new ArgumentException("One site can own at most one site link.", nameof(edits));
}
var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "wbsetsitelink",
token = WikiSiteToken.Edit,
id = Id,
@new = Id == null ? FormatEntityType(Type) : null,
baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null,
bot = isBot,
summary = summary,
linksite = siteGroup.Key,
linktitle = link,
badges = badges,
}), cancellationToken);
LoadEntityMinimal(jresult["entity"]);
if (!strict)
{
checkbaseRev = false;
}
}
break;
}
case nameof(Claims):
foreach (var entry in prop.Where(e => e.State == EntityEditEntryState.Updated))
{
var value = (Claim)entry.Value;
var claimContract = value.ToContract(false);
if (value.Id == null)
{
// New claim. We need to assign an ID manually.
// https://phabricator.wikimedia.org/T182573#3828344
if (Id == null)
{
// This is a new entity, so we need to create it first.
var jresult1 = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "wbeditentity",
token = WikiSiteToken.Edit,
@new = FormatEntityType(Type),
bot = isBot,
summary = (string)null,
data = "{}"
}), cancellationToken);
if (!strict)
{
checkbaseRev = false;
}
LoadEntityMinimal(jresult1["entity"]);
}
claimContract.Id = Utility.NewClaimGuid(Id);
}
var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "wbsetclaim",
token = WikiSiteToken.Edit,
@new = Id == null ? FormatEntityType(Type) : null,
baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null,
bot = isBot,
summary = summary,
claim = Utility.WikiJsonSerializer.Serialize(claimContract),
}), cancellationToken);
// jresult["claim"] != null
LastRevisionId = (int)jresult["pageinfo"]["lastrevid"];
if (!strict)
{
checkbaseRev = false;
}
}
foreach (var batch in prop.Where(e => e.State == EntityEditEntryState.Removed)
.Select(e => ((Claim)e.Value).Id).Partition(50))
{
var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "wbremoveclaims",
token = WikiSiteToken.Edit,
id = Id,
@new = Id == null ? FormatEntityType(Type) : null,
baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null,
bot = isBot,
summary = summary,
claim = MediaWikiHelper.JoinValues(batch),
}), cancellationToken);
LastRevisionId = (int)jresult["pageinfo"]["lastrevid"];
if (!strict)
{
checkbaseRev = false;
}
}
break;
default:
throw new ArgumentException($"Unrecognized {nameof(Entity)} property name: {prop.Key}.");
}
}
void LoadEntityMinimal(JToken jentity)
{
Debug.Assert(jentity != null);
Id = (string)jentity["id"];
Type = SerializableEntity.ParseEntityType((string)jentity["type"]);
LastRevisionId = (int)jentity["lastrevid"];
}
}
/// <summary>
/// Construct a sequence of <see cref="WikiPageStub"/> from the given page IDs.
/// </summary>
/// <param name="site">The site in which to query for the pages.</param>
/// <param name="ids">The page IDs to query.</param>
/// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="ids"/> is <c>null</c>.</exception>
/// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns>
/// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks>
public static async IAsyncEnumerable <WikiPageStub> FromPageIds(WikiSite site, IEnumerable <int> ids,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
? 500
: 50;
foreach (var partition in ids.Partition(titleLimit))
{
var jresult = await site.InvokeMediaWikiApiAsync(
new MediaWikiFormRequestMessage(new { action = "query", pageids = MediaWikiHelper.JoinValues(partition), }), cancellationToken);
Debug.Assert(jresult["query"] != null);
var jpages = jresult["query"]["pages"];
using (ExecutionContextStash.Capture())
foreach (var id in partition)
{
var jpage = jpages[id.ToString(CultureInfo.InvariantCulture)];
if (jpage["missing"] == null)
{
yield return(new WikiPageStub(id, (string)jpage["title"], (int)jpage["ns"]));
}
else
{
yield return(new WikiPageStub(id, MissingPageTitle, UnknownNamespaceId));
}
}
}
}
/// <summary>
/// Enumerate transcluded pages trans from the page.
/// </summary>
public static IAsyncEnumerable <string> EnumTransclusionsAsync(WikiSite site, string titlesExpr, IEnumerable <int> namespaces = null, IEnumerable <string> transcludedTitlesExpr = null, int limit = -1)
{
// transcludedTitlesExpr should be full titles with ns prefix.
var pa = new Dictionary <string, object>
{
{ "action", "query" }, { "prop", "templates" }, { "tllimit", limit > 0 ? limit : site.ListingPagingSize }, { "tlnamespace", namespaces == null ? null : MediaWikiHelper.JoinValues(namespaces) }, { "tltemplates", transcludedTitlesExpr == null ? null : MediaWikiHelper.JoinValues(transcludedTitlesExpr) }
};
pa["titles"] = titlesExpr;
var resultCounter = 0;
return(QueryWithContinuation(site, pa, null)
.SelectMany(jpages =>
{
var page = jpages.Values().First();
var links = (JArray)page?["templates"];
if (links != null)
{
resultCounter += links.Count;
site.Logger.LogDebug("Loaded {Count} items transcluded by [[{Title}]] on {Site}.",
resultCounter, titlesExpr, site);
return links.Select(l => (string)l["title"]).ToAsyncEnumerable();
}
return AsyncEnumerable.Empty <string>();
}));
}
/// <summary>
/// Enumerate links from the page.
/// </summary>
public static IAsyncEnumerable <string> EnumLinksAsync(WikiSite site, string titlesExpr, /* optional */ IEnumerable <int> namespaces)
{
var pa = new Dictionary <string, object>
{
{ "action", "query" }, { "prop", "links" }, { "pllimit", site.ListingPagingSize }, { "plnamespace", namespaces == null ? null : MediaWikiHelper.JoinValues(namespaces) },
};
pa["titles"] = titlesExpr;
var resultCounter = 0;
return(QueryWithContinuation(site, pa, null)
.SelectMany(jpages =>
{
var page = jpages.Values().First();
var links = (JArray)page?["links"];
if (links != null)
{
resultCounter += links.Count;
site.Logger.LogDebug("Loaded {Count} items linking to [[{Title}]] on {Site}.", resultCounter, titlesExpr, site);
return links.Select(l => (string)l["title"]).ToAsyncEnumerable();
}
return AsyncEnumerable.Empty <string>();
}));
}
/// <summary>
/// Asynchronously purges the pages.
/// </summary>
/// <returns>A collection of pages that haven't been successfully purged, because of either missing or invalid titles.</returns>
public static async Task <IReadOnlyCollection <PurgeFailureInfo> > PurgePagesAsync(IEnumerable <WikiPage> pages, PagePurgeOptions options, CancellationToken cancellationToken)
{
if (pages == null)
{
throw new ArgumentNullException(nameof(pages));
}
List <PurgeFailureInfo> failedPages = null;
// You can even purge pages from different sites.
foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p)))
{
var site = sitePages.Key.Site;
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
? 500
: 50;
using (site.BeginActionScope(sitePages, options))
{
foreach (var partition in sitePages.Partition(titleLimit).Select(partition => partition.ToList()))
{
string titles;
string ids;
if (sitePages.Key.HasTitle)
{
// If a page has both title and ID information,
// we will use title anyway.
site.Logger.LogDebug("Purging {Count} pages by title.", partition.Count);
titles = MediaWikiHelper.JoinValues(partition.Select(p => p.Title));
ids = null;
}
else
{
site.Logger.LogDebug("Purging {Count} pages by ID.", partition.Count);
Debug.Assert(sitePages.All(p => p.PageStub.HasId));
titles = null;
ids = MediaWikiHelper.JoinValues(partition.Select(p => p.Id));
}
try
{
var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new
{
action = "purge",
titles = titles,
pageids = ids,
forcelinkupdate = (options & PagePurgeOptions.ForceLinkUpdate) == PagePurgeOptions.ForceLinkUpdate,
forcerecursivelinkupdate = (options & PagePurgeOptions.ForceRecursiveLinkUpdate) == PagePurgeOptions.ForceRecursiveLinkUpdate,
}), cancellationToken);
// Now check whether the pages have been purged successfully.
foreach (var jitem in jresult["purge"])
{
if (jitem["missing"] != null || jitem["invalid"] != null)
{
if (failedPages == null)
{
failedPages = new List <PurgeFailureInfo>();
}
failedPages.Add(new PurgeFailureInfo(MediaWikiHelper.PageStubFromJson((JObject)jitem), (string)jitem["invalidreason"]));
}
}
}
catch (OperationFailedException ex)
{
if (ex.ErrorCode == "cantpurge")
{
throw new UnauthorizedOperationException(ex);
}
throw;
}
}
}
}
return(failedPages ?? emptyPurgeFailures);
}
/// <summary>
/// Refresh a sequence of pages.
/// </summary>
public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, IWikiPageQueryProvider options, CancellationToken cancellationToken)
{
if (pages == null)
{
throw new ArgumentNullException(nameof(pages));
}
// You can even fetch pages from different sites.
foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p)))
{
var site = sitePages.Key.Site;
var queryParams = options.EnumParameters(site.SiteInfo.Version).ToDictionary();
var titleLimit = options.GetMaxPaginationSize(site.SiteInfo.Version, site.AccountInfo.HasRight(UserRights.ApiHighLimits));
using (site.BeginActionScope(sitePages, options))
{
foreach (var partition in sitePages.Partition(titleLimit))
{
if (sitePages.Key.HasTitle)
{
// If a page has both title and ID information,
// we will use title anyway.
site.Logger.LogDebug("Fetching {Count} pages by title.", partition.Count);
queryParams["titles"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Title));
}
else
{
site.Logger.LogDebug("Fetching {Count} pages by ID.", partition.Count);
Debug.Assert(sitePages.All(p => p.PageStub.HasId));
queryParams["pageids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id));
}
var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken);
var jquery = (JObject)jobj["query"];
var continuationStatus = ParseContinuationParameters(jobj, queryParams, null);
// Process continuation caused by props (e.g. langlinks) that contain a list that is too long.
if (continuationStatus != CONTINUATION_DONE)
{
var queryParams1 = new Dictionary <string, object>();
var continuationParams = new Dictionary <string, object>();
var jobj1 = jobj;
ParseContinuationParameters(jobj1, queryParams1, continuationParams);
while (continuationStatus != CONTINUATION_DONE)
{
if (continuationStatus == CONTINUATION_LOOP)
{
throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop);
}
Debug.Assert(continuationStatus == CONTINUATION_AVAILABLE);
site.Logger.LogDebug("Detected query continuation. PartitionCount={PartitionCount}.", partition.Count);
queryParams1.Clear();
queryParams1.MergeFrom(queryParams);
queryParams1.MergeFrom(continuationParams);
jobj1 = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams1), cancellationToken);
var jquery1 = jobj1["query"];
if (jquery1.HasValues)
{
// Merge JSON response
jquery.Merge(jquery1);
}
continuationStatus = ParseContinuationParameters(jobj1, queryParams1, continuationParams);
}
}
if (sitePages.Key.HasTitle)
{
// Process title normalization.
var normalized = jquery["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]);
// Process redirects.
var redirects = jquery["redirects"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]);
var pageInfoDict = ((JObject)jquery["pages"]).Properties()
.ToDictionary(p => (string)p.Value["title"]);
foreach (var page in partition)
{
var title = page.Title;
// Normalize the title first.
if (normalized?.ContainsKey(title) ?? false)
{
title = normalized[title];
}
// Then process the redirects.
var redirectTrace = new List <string>();
while (redirects?.ContainsKey(title) ?? false)
{
redirectTrace.Add(title); // Adds the last title
var next = redirects[title];
if (redirectTrace.Contains(next))
{
throw new InvalidOperationException(string.Format(Prompts.ExceptionWikiPageResolveCircularRedirect1, string.Join("->", redirectTrace)));
}
title = next;
}
// Finally, get the page.
var pageInfo = pageInfoDict[title];
if (redirectTrace.Count > 0)
{
page.RedirectPath = redirectTrace;
}
MediaWikiHelper.PopulatePageFromJson(page, (JObject)pageInfo.Value, options);
}
}
else
{
foreach (var page in partition)
{
var jPage = (JObject)jquery["pages"][page.Id.ToString(CultureInfo.InvariantCulture)];
MediaWikiHelper.PopulatePageFromJson(page, jPage, options);
}
}
}
}
}
}
