/// <summary> /// Construct a sequence of <see cref="WikiPageStub"/> from the given page IDs. /// </summary> /// <param name="site">The site in which to query for the pages.</param> /// <param name="ids">The page IDs to query.</param> /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="ids"/> is <c>null</c>.</exception> /// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns> /// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks> public static IAsyncEnumerable <WikiPageStub> FromPageIds(WikiSite site, IEnumerable <int> ids) { return(AsyncEnumerableFactory.FromAsyncGenerator <WikiPageStub>(async(sink, ct) => { var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; foreach (var partition in ids.Partition(titleLimit)) { var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "query", pageids = MediaWikiHelper.JoinValues(partition), }), ct); Debug.Assert(jresult["query"] != null); var jpages = jresult["query"]["pages"]; foreach (var id in partition) { var jpage = jpages[id.ToString(CultureInfo.InvariantCulture)]; if (jpage["missing"] == null) { sink.Yield(new WikiPageStub(id, (string)jpage["title"], (int)jpage["ns"])); } else { sink.Yield(new WikiPageStub(id, MissingPageTitle, UnknownNamespaceId)); } } await sink.Wait(); } })); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object> > EnumParameters(MediaWikiVersion version) { return(new OrderedKeyValuePairs <string, object> { { "ppprop", SelectedProperties == null ? null : MediaWikiHelper.JoinValues(SelectedProperties) } }); }
private IEnumerable <KeyValuePair <string, object> > EnumParams(bool isList) { var prefix = isList ? null : "g"; var dict = new Dictionary <string, object> { { prefix + "rcdir", TimeAscending ? "newer" : "older" }, { prefix + "rcstart", StartTime }, { prefix + "rcend", EndTime }, { prefix + "rcnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) }, { prefix + "rcuser", UserName }, { prefix + "rcexcludeuser", ExcludedUserName }, { prefix + "rctag", Tag }, { prefix + "rctype", ParseRecentChangesTypes(TypeFilters) }, { prefix + "rcshow", ParseFilters() }, { prefix + "rctoponly", LastRevisionsOnly }, { prefix + "rclimit", PaginationSize } }; if (isList) { var fields = "user|userid|comment|parsedcomment|flags|timestamp|title|ids|sizes|redirect|loginfo|tags|sha1"; if (Site.AccountInfo.HasRight(UserRights.Patrol)) { fields += "|patrolled"; } dict.Add("rcprop", fields); } return(dict); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object> > EnumListParameters() { if ((TargetTitle != null) == (TargetPageId != null)) { throw new ArgumentException(string.Format(Prompts.ExceptionArgumentExpectEitherNull2, nameof(TargetTitle), nameof(TargetPageId))); } var actualPaginationSize = PaginationSize; if (AllowRedirectedLinks) { // When the blredirect parameter is set, this module behaves slightly differently. // bllimit applies to both levels separately: if e.g. bllimit=10, // at most 10 first-level pages (pages that link to bltitle) and // 10 second-level pages (pages that link to bltitle through a redirect) will be listed. // Continuing queries also works differently. actualPaginationSize = Math.Max(1, PaginationSize / 2); } return(new Dictionary <string, object> { { "bltitle", TargetTitle }, { "blpageid", TargetPageId }, { "blnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) }, { "blfilterredir", RedirectsFilter.ToString("redirects", "nonredirects") }, { "bllimit", actualPaginationSize }, { "blredirect", AllowRedirectedLinks } }); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object> > EnumListParameters() { var dict = new Dictionary <string, object> { { "srsearch", Keyword }, { "srnamespace", NamespaceIds == null ? "*" : MediaWikiHelper.JoinValues(NamespaceIds) }, { "srwhat", MatchingField }, { "srlimit", PaginationSize }, { "srinterwiki", IncludesInterwiki }, { "srbackend", BackendName } }; // Include redirect pages in the search. From 1.23 onwards, redirects are always included. (Removed in 1.23) if (Site.SiteInfo.Version < new MediaWikiVersion(1, 23)) { dict["srredirects"] = true; } dict["srwhat"] = MatchingField switch { SearchableField.Title => "title", SearchableField.Text => "text", SearchableField.NearMatch => "nearmatch", _ => throw new ArgumentOutOfRangeException() }; return(dict); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object?> > EnumListParameters() { return(new Dictionary <string, object> { { "tlnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) }, { "tllimit", PaginationSize }, { "tltemplates", MatchingTitles == null ? null : MediaWikiHelper.JoinValues(MatchingTitles) }, { "tldir", OrderDescending ? "descending" : "ascending" } }); }
public async Task Issue67() { var site = await WpEnSiteAsync; var items = await new CategoriesGenerator(site) { PageTitle = MediaWikiHelper.JoinValues(new[] { "Test", ".test", "Test_(Unix)", "Test_(assessment)" }), }.EnumItemsAsync().ToListAsync(); ShallowTrace(items); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object> > EnumListParameters() { return(new Dictionary <string, object> { { "eititle", TargetTitle }, { "einamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) }, { "eifilterredir", RedirectsFilter.ToString("redirects", "nonredirects") }, { "eilimit", PaginationSize } }); }
/// <summary> /// Refresh a sequence of revisions by revid, along with their owner pages. /// </summary> /// <remarks> /// <para>If there's invalid revision id in <paramref name="revIds"/>, an <see cref="ArgumentException"/> will be thrown while enumerating.</para> /// </remarks> public static IAsyncEnumerable <Revision> FetchRevisionsAsync(WikiSite site, IEnumerable <int> revIds, IWikiPageQueryProvider options, CancellationToken cancellationToken) { if (revIds == null) { throw new ArgumentNullException(nameof(revIds)); } var queryParams = options.EnumParameters(site.SiteInfo.Version).ToDictionary(); // Remove any rvlimit magic word generated by RevisionsPropertyProvider. // We are only fetching by revisions. queryParams.Remove("rvlimit"); var titleLimit = options.GetMaxPaginationSize(site.SiteInfo.Version, site.AccountInfo.HasRight(UserRights.ApiHighLimits)); return(AsyncEnumerableFactory.FromAsyncGenerator <Revision>(async sink => { // Page ID --> Page Stub var stubDict = new Dictionary <int, WikiPageStub>(); var revDict = new Dictionary <int, Revision>(); using (site.BeginActionScope(null, (object)revIds)) { foreach (var partition in revIds.Partition(titleLimit)) { site.Logger.LogDebug("Fetching {Count} revisions from {Site}.", partition.Count, site); queryParams["revids"] = MediaWikiHelper.JoinValues(partition); var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken); var jpages = (JObject)jobj["query"]["pages"]; // Generate stubs first foreach (var p in jpages) { var jrevs = p.Value["revisions"]; if (jrevs == null || !jrevs.HasValues) { continue; } var id = Convert.ToInt32(p.Key); if (!stubDict.TryGetValue(id, out var stub)) { stub = new WikiPageStub(id, (string)p.Value["title"], (int)p.Value["ns"]); stubDict.Add(id, stub); } foreach (var jrev in jrevs) { var rev = jrev.ToObject <Revision>(Utility.WikiJsonSerializer); rev.Page = stub; revDict.Add(rev.Id, rev); } } await sink.YieldAndWait(partition.Select(id => revDict.TryGetValue(id, out var rev) ? rev : null)); } } })); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object> > EnumParameters(MediaWikiVersion version) { var p = new OrderedKeyValuePairs <string, object> { { "lhprop", "pageid|title|redirect" }, { "lhshow", RedirectFilter.ToString("redirect", "!redirect", null) } }; if (NamespaceSelection != null) { p.Add("lhnamespace", MediaWikiHelper.JoinValues(NamespaceSelection)); } return(p); }
/// <inheritdoc/> public override IEnumerable <KeyValuePair <string, object> > EnumListParameters() { if (string.IsNullOrEmpty(CategoryTitle)) { throw new InvalidOperationException(string.Format(Prompts.ExceptionArgumentIsEmpty1, nameof(CategoryTitle))); } return(new Dictionary <string, object> { { "cmtitle", CategoryTitle }, { "cmlimit", PaginationSize }, { "cmnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) }, { "cmtype", ParseMemberTypes(MemberTypes) } }); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object> > EnumParameters(MediaWikiVersion version) { var p = new OrderedKeyValuePairs <string, object> { { "clprop", "sortkey|timestamp|hidden" }, { "clshow", HiddenCategoryFilter.ToString("hidden", "!hidden", null) } }; if (CategorySelection != null) { p.Add("clcategories", MediaWikiHelper.JoinValues(CategorySelection)); } return(p); }
public static async Task RefreshEntitiesAsync(IEnumerable <Entity> entities, EntityQueryOptions options, IEnumerable <string> languages, CancellationToken cancellationToken) { if (entities == null) { throw new ArgumentNullException(nameof(entities)); } var langs = languages == null ? null : MediaWikiHelper.JoinValues(languages); if (string.IsNullOrEmpty(langs)) { langs = null; } // You can even fetch pages from different sites. foreach (var siteEntities in entities.GroupBy(p => p.Site)) { var site = siteEntities.Key; var req = BuildQueryOptions(langs, options); req["action"] = "wbgetentities"; var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; using (site.BeginActionScope(entities, options)) { foreach (var partition in siteEntities.Partition(titleLimit).Select(partition => partition.ToList())) { //site.Logger.LogDebug("Fetching {Count} pages from {Site}.", partition.Count, site); // We use ids to query pages. req["ids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id)); var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(req), cancellationToken); var jentities = (JObject)jresult["entities"]; foreach (var entity in partition) { var jentity = jentities[entity.Id]; // We can write Q123456 as q123456 in query params, but server will return Q123456 anyway. if (jentity == null) { jentity = jentities.Properties().FirstOrDefault(p => string.Equals(p.Name, entity.Id, StringComparison.OrdinalIgnoreCase)); } if (jentity == null) { throw new UnexpectedDataException($"Cannot find the entity with id {entity.Id} in the response."); } entity.LoadFromJson(jentity, options, false); } } } } }
/// <summary> /// Construct a sequence of <see cref="WikiPageStub"/> from the given page titles. /// </summary> /// <param name="site">The site in which to query for the pages.</param> /// <param name="titles">The page IDs to query.</param> /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="titles"/> is <c>null</c>.</exception> /// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns> /// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks> public static IAsyncEnumerable <WikiPageStub> FromPageTitles(WikiSite site, IEnumerable <string> titles) { if (site == null) { throw new ArgumentNullException(nameof(site)); } if (titles == null) { throw new ArgumentNullException(nameof(titles)); } return(AsyncEnumerableFactory.FromAsyncGenerator <WikiPageStub>(async(sink, ct) => { var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; foreach (var partition in titles.Partition(titleLimit)) { var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "query", titles = MediaWikiHelper.JoinValues(partition), }), ct); Debug.Assert(jresult["query"] != null); // Process title normalization. var normalizedDict = jresult["query"]["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); var pageDict = ((JObject)jresult["query"]["pages"]).Properties() .ToDictionary(p => (string)p.Value["title"], p => p.Value); foreach (var name in partition) { if (normalizedDict == null || !normalizedDict.TryGetValue(name, out var normalizedName)) { normalizedName = name; } var jpage = pageDict[normalizedName]; if (jpage["missing"] == null) { sink.Yield(new WikiPageStub((int)jpage["pageid"], (string)jpage["title"], (int)jpage["ns"])); } else { sink.Yield(new WikiPageStub(MissingPageIdMask, (string)jpage["title"], (int)jpage["ns"])); } } await sink.Wait(); } })); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object> > EnumParameters(MediaWikiVersion version) { var p = new OrderedKeyValuePairs <string, object> { { "rvprop", FetchContent ? "ids|timestamp|flags|comment|user|userid|contentmodel|sha1|tags|size|content" : "ids|timestamp|flags|comment|user|userid|contentmodel|sha1|tags|size" } }; if (Slots != null || version >= new MediaWikiVersion(1, 32)) { // If user specified Slots explicitly, then we will respect it regardless of MW version. p.Add("rvslots", Slots == null ? RevisionSlot.MainSlotName : MediaWikiHelper.JoinValues(Slots)); } return(p); }
/// <inheritdoc/> public override IEnumerable <KeyValuePair <string, object> > EnumListParameters() { var dict = new Dictionary <string, object> { { "rnnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) }, { "rnlimit", PaginationSize }, }; if (Site.SiteInfo.Version >= new Version(1, 26)) { dict.Add("rnfilterredir", RedirectsFilter.ToString("redirects", "nonredirects")); } else if (RedirectsFilter == PropertyFilterOption.WithProperty) { dict.Add("rnredirect", true); // for MW 1.26-, we cannot really implement RedirectsFilter == PropertyFilterOption.WithoutProperty } return(dict); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object> > EnumListParameters() { var prop = new Dictionary <string, object> { { "gsradius", Radius }, { "gsnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) }, { "gsprimary", IncludesSecondaryCoordinates ? "all" : "primary" }, { "gsglobe", TargetCoordinate.Globe }, { "gslimit", PaginationSize }, }; if (TargetTitle != null) { // When searching by page title, it would be better for MW API to // assume `gsglobe` corresponds to the `globe` of that page, // but there is currently no such behavior. prop["gspage"] = TargetTitle; } else if (!BoundingRectangle.IsEmpty) { var rect = BoundingRectangle; // This is way too large than the width acceptable by MW API. // And this causes weird `Right` value. if (rect.Width > 180) { throw new ArgumentException("Bounding box is too big.", nameof(BoundingRectangle)); } rect.Normalize(); var right = rect.Right; if (right > 180) { right -= 360; } prop["gsbbox"] = string.Format(CultureInfo.InvariantCulture, "{0}|{1}|{2}|{3}", rect.Top, rect.Left, rect.Bottom, right); } else { prop["gscoord"] = TargetCoordinate.Latitude.ToString(CultureInfo.InvariantCulture) + "|" + TargetCoordinate.Longitude.ToString(CultureInfo.InvariantCulture); } return(prop); }
public static IAsyncEnumerable <string> EntityIdsFromSiteLinksAsync(WikiSite site, string siteName, IEnumerable <string> siteLinks) { Debug.Assert(siteName != null); Debug.Assert(siteLinks != null); var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; return(AsyncEnumerableFactory.FromAsyncGenerator <string>(async(sink, ct) => { var req = new OrderedKeyValuePairs <string, string> { { "action", "wbgetentities" }, { "props", "sitelinks" }, { "sites", siteName }, { "sitefilter", siteName }, }; using (site.BeginActionScope(siteLinks)) { foreach (var partition in siteLinks.Partition(titleLimit).Select(partition => partition.ToList())) { //site.Logger.LogDebug("Fetching {Count} pages from {Site}.", partition.Count, site); for (int i = 0; i < partition.Count; i++) { if (partition[i] == null) { throw new ArgumentException("Link titles contain null element.", nameof(siteLinks)); } // Do some basic title normalization locally. // Note Wikibase cannot even normalize the first letter case of the title. partition[i] = partition[i].Trim(whitespaceAndUnderscore).Replace('_', ' '); } req["titles"] = MediaWikiHelper.JoinValues(partition); var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(req), ct); var jentities = (JObject)jresult["entities"]; var nameIdDict = jentities.PropertyValues().Where(e => e["missing"] == null) .ToDictionary(e => (string)e["sitelinks"][siteName]["title"], e => (string)e["id"]); await sink.YieldAndWait(partition.Select(title => nameIdDict.TryGetValue(title, out var id) ? id : null)); } } })); }
/// <inheritdoc /> public override IEnumerable <KeyValuePair <string, object> > EnumListParameters() { var prop = new Dictionary <string, object> { { "gsradius", Radius }, { "gsnamespace", NamespaceIds == null ? null : MediaWikiHelper.JoinValues(NamespaceIds) }, { "gsprimary", IncludesSecondaryCoordinates ? "all" : "primary" }, { "gsglobe", TargetCoordinate.Globe }, { "gslimit", PaginationSize }, }; if (TargetTitle != null) { // When searching by page title, it would be better for MW API to // assume `gsglobe` corresponds to the `globe` of that page, // but there is currently no such behavior. prop["gspage"] = TargetTitle; } else { prop["gscoord"] = TargetCoordinate.Latitude + "|" + TargetCoordinate.Longitude; } return(prop); }
/// <inheritdoc/> public override string ToString() { var tags = Tags == null ? null : MediaWikiHelper.JoinValues(Tags); return($"Revision#{Id}, {Flags}, {tags}, SHA1={Sha1}"); }
/// <summary> /// Refresh a sequence of pages. /// </summary> public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, IWikiPageQueryProvider options, CancellationToken cancellationToken) { if (pages == null) { throw new ArgumentNullException(nameof(pages)); } // You can even fetch pages from different sites. foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p))) { var site = sitePages.Key.Site; var queryParams = options.EnumParameters().ToDictionary(); var titleLimit = options.GetMaxPaginationSize(site.AccountInfo.HasRight(UserRights.ApiHighLimits)); using (site.BeginActionScope(sitePages, options)) { foreach (var partition in sitePages.Partition(titleLimit)) { if (sitePages.Key.HasTitle) { // If a page has both title and ID information, // we will use title anyway. site.Logger.LogDebug("Fetching {Count} pages by title.", partition.Count); queryParams["titles"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Title)); } else { site.Logger.LogDebug("Fetching {Count} pages by ID.", partition.Count); Debug.Assert(sitePages.All(p => p.PageStub.HasId)); queryParams["pageids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id)); } // For single-page fetching, force fetching 1 revision only. if (partition.Count == 1) { queryParams["rvlimit"] = 1; } else { queryParams.Remove("rvlimit"); } var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken); if (sitePages.Key.HasTitle) { // Process title normalization. var normalized = jobj["query"]["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); // Process redirects. var redirects = jobj["query"]["redirects"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); var pageInfoDict = ((JObject)jobj["query"]["pages"]).Properties() .ToDictionary(p => (string)p.Value["title"]); foreach (var page in partition) { var title = page.Title; // Normalize the title first. if (normalized?.ContainsKey(title) ?? false) { title = normalized[title]; } // Then process the redirects. var redirectTrace = new List <string>(); while (redirects?.ContainsKey(title) ?? false) { redirectTrace.Add(title); // Adds the last title var next = redirects[title]; if (redirectTrace.Contains(next)) { throw new InvalidOperationException($"Cannot resolve circular redirect: {string.Join("->", redirectTrace)}."); } title = next; } // Finally, get the page. var pageInfo = pageInfoDict[title]; if (redirectTrace.Count > 0) { page.RedirectPath = redirectTrace; } MediaWikiHelper.PopulatePageFromJson(page, (JObject)pageInfo.Value, options); } } else { foreach (var page in partition) { var jPage = (JObject)jobj["query"]["pages"][page.Id.ToString()]; MediaWikiHelper.PopulatePageFromJson(page, jPage, options); } } } } } }
private async Task ProgressiveEditAsync(IEnumerable <EntityEditEntry> edits, string summary, bool isBot, bool strict, CancellationToken cancellationToken) { Debug.Assert(edits != null); var checkbaseRev = true; foreach (var prop in edits.GroupBy(e => e.PropertyName)) { if (prop.Any(p => p.Value == null)) { throw new ArgumentException($"Detected null value in {prop} entries.", nameof(edits)); } switch (prop.Key) { case nameof(DataType): throw new NotSupportedException("Setting data type is not possible in progressive mode."); case nameof(Labels): foreach (var p in prop) { var value = (WbMonolingualText)p.Value; var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "wbsetlabel", token = WikiSiteToken.Edit, id = Id, @new = Id == null ? FormatEntityType(Type) : null, baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null, bot = isBot, summary = summary, language = value.Language, value = p.State == EntityEditEntryState.Updated ? value.Text : null, }), cancellationToken); LoadEntityMinimal(jresult["entity"]); if (!strict) { checkbaseRev = false; } } break; case nameof(Descriptions): foreach (var p in prop) { var value = (WbMonolingualText)p.Value; var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "wbsetdescription", token = WikiSiteToken.Edit, id = Id, @new = Id == null ? FormatEntityType(Type) : null, baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null, bot = isBot, summary = summary, language = value.Language, value = p.State == EntityEditEntryState.Updated ? value.Text : null, }), cancellationToken); LoadEntityMinimal(jresult["entity"]); if (!strict) { checkbaseRev = false; } } break; case nameof(Aliases): { var entries = prop.GroupBy(t => ((WbMonolingualText)t.Value).Language); foreach (var langGroup in entries) { var addExpr = MediaWikiHelper.JoinValues(langGroup .Where(e => e.State == EntityEditEntryState.Updated) .Select(e => ((WbMonolingualText)e.Value).Text)); var removeExpr = MediaWikiHelper.JoinValues(langGroup .Where(e => e.State == EntityEditEntryState.Removed) .Select(e => ((WbMonolingualText)e.Value).Text)); var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "wbsetaliases", token = WikiSiteToken.Edit, id = Id, @new = Id == null ? FormatEntityType(Type) : null, baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null, bot = isBot, summary = summary, language = langGroup.Key, add = addExpr.Length == 0 ? null : addExpr, remove = removeExpr.Length == 0 ? null : removeExpr, }), cancellationToken); LoadEntityMinimal(jresult["entity"]); if (!strict) { checkbaseRev = false; } } break; } case nameof(SiteLinks): { var entries = prop.GroupBy(t => ((EntitySiteLink)t.Value).Site); foreach (var siteGroup in entries) { string link = null, badges = null; try { var item = siteGroup.Single(); if (item.State == EntityEditEntryState.Updated) { var value = (EntitySiteLink)item.Value; link = value.Title; badges = MediaWikiHelper.JoinValues(value.Badges); } } catch (InvalidOperationException) { throw new ArgumentException("One site can own at most one site link.", nameof(edits)); } var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "wbsetsitelink", token = WikiSiteToken.Edit, id = Id, @new = Id == null ? FormatEntityType(Type) : null, baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null, bot = isBot, summary = summary, linksite = siteGroup.Key, linktitle = link, badges = badges, }), cancellationToken); LoadEntityMinimal(jresult["entity"]); if (!strict) { checkbaseRev = false; } } break; } case nameof(Claims): foreach (var entry in prop.Where(e => e.State == EntityEditEntryState.Updated)) { var value = (Claim)entry.Value; var claimContract = value.ToContract(false); if (value.Id == null) { // New claim. We need to assign an ID manually. // https://phabricator.wikimedia.org/T182573#3828344 if (Id == null) { // This is a new entity, so we need to create it first. var jresult1 = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "wbeditentity", token = WikiSiteToken.Edit, @new = FormatEntityType(Type), bot = isBot, summary = (string)null, data = "{}" }), cancellationToken); if (!strict) { checkbaseRev = false; } LoadEntityMinimal(jresult1["entity"]); } claimContract.Id = Utility.NewClaimGuid(Id); } var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "wbsetclaim", token = WikiSiteToken.Edit, @new = Id == null ? FormatEntityType(Type) : null, baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null, bot = isBot, summary = summary, claim = Utility.WikiJsonSerializer.Serialize(claimContract), }), cancellationToken); // jresult["claim"] != null LastRevisionId = (int)jresult["pageinfo"]["lastrevid"]; if (!strict) { checkbaseRev = false; } } foreach (var batch in prop.Where(e => e.State == EntityEditEntryState.Removed) .Select(e => ((Claim)e.Value).Id).Partition(50)) { var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "wbremoveclaims", token = WikiSiteToken.Edit, id = Id, @new = Id == null ? FormatEntityType(Type) : null, baserevid = checkbaseRev && LastRevisionId > 0 ? (int?)LastRevisionId : null, bot = isBot, summary = summary, claim = MediaWikiHelper.JoinValues(batch), }), cancellationToken); LastRevisionId = (int)jresult["pageinfo"]["lastrevid"]; if (!strict) { checkbaseRev = false; } } break; default: throw new ArgumentException($"Unrecognized {nameof(Entity)} property name: {prop.Key}."); } } void LoadEntityMinimal(JToken jentity) { Debug.Assert(jentity != null); Id = (string)jentity["id"]; Type = SerializableEntity.ParseEntityType((string)jentity["type"]); LastRevisionId = (int)jentity["lastrevid"]; } }
/// <summary> /// Construct a sequence of <see cref="WikiPageStub"/> from the given page IDs. /// </summary> /// <param name="site">The site in which to query for the pages.</param> /// <param name="ids">The page IDs to query.</param> /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="ids"/> is <c>null</c>.</exception> /// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns> /// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks> public static async IAsyncEnumerable <WikiPageStub> FromPageIds(WikiSite site, IEnumerable <int> ids, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; foreach (var partition in ids.Partition(titleLimit)) { var jresult = await site.InvokeMediaWikiApiAsync( new MediaWikiFormRequestMessage(new { action = "query", pageids = MediaWikiHelper.JoinValues(partition), }), cancellationToken); Debug.Assert(jresult["query"] != null); var jpages = jresult["query"]["pages"]; using (ExecutionContextStash.Capture()) foreach (var id in partition) { var jpage = jpages[id.ToString(CultureInfo.InvariantCulture)]; if (jpage["missing"] == null) { yield return(new WikiPageStub(id, (string)jpage["title"], (int)jpage["ns"])); } else { yield return(new WikiPageStub(id, MissingPageTitle, UnknownNamespaceId)); } } } }
/// <summary> /// Enumerate transcluded pages trans from the page. /// </summary> public static IAsyncEnumerable <string> EnumTransclusionsAsync(WikiSite site, string titlesExpr, IEnumerable <int> namespaces = null, IEnumerable <string> transcludedTitlesExpr = null, int limit = -1) { // transcludedTitlesExpr should be full titles with ns prefix. var pa = new Dictionary <string, object> { { "action", "query" }, { "prop", "templates" }, { "tllimit", limit > 0 ? limit : site.ListingPagingSize }, { "tlnamespace", namespaces == null ? null : MediaWikiHelper.JoinValues(namespaces) }, { "tltemplates", transcludedTitlesExpr == null ? null : MediaWikiHelper.JoinValues(transcludedTitlesExpr) } }; pa["titles"] = titlesExpr; var resultCounter = 0; return(QueryWithContinuation(site, pa, null) .SelectMany(jpages => { var page = jpages.Values().First(); var links = (JArray)page?["templates"]; if (links != null) { resultCounter += links.Count; site.Logger.LogDebug("Loaded {Count} items transcluded by [[{Title}]] on {Site}.", resultCounter, titlesExpr, site); return links.Select(l => (string)l["title"]).ToAsyncEnumerable(); } return AsyncEnumerable.Empty <string>(); })); }
/// <summary> /// Enumerate links from the page. /// </summary> public static IAsyncEnumerable <string> EnumLinksAsync(WikiSite site, string titlesExpr, /* optional */ IEnumerable <int> namespaces) { var pa = new Dictionary <string, object> { { "action", "query" }, { "prop", "links" }, { "pllimit", site.ListingPagingSize }, { "plnamespace", namespaces == null ? null : MediaWikiHelper.JoinValues(namespaces) }, }; pa["titles"] = titlesExpr; var resultCounter = 0; return(QueryWithContinuation(site, pa, null) .SelectMany(jpages => { var page = jpages.Values().First(); var links = (JArray)page?["links"]; if (links != null) { resultCounter += links.Count; site.Logger.LogDebug("Loaded {Count} items linking to [[{Title}]] on {Site}.", resultCounter, titlesExpr, site); return links.Select(l => (string)l["title"]).ToAsyncEnumerable(); } return AsyncEnumerable.Empty <string>(); })); }
/// <summary> /// Asynchronously purges the pages. /// </summary> /// <returns>A collection of pages that haven't been successfully purged, because of either missing or invalid titles.</returns> public static async Task <IReadOnlyCollection <PurgeFailureInfo> > PurgePagesAsync(IEnumerable <WikiPage> pages, PagePurgeOptions options, CancellationToken cancellationToken) { if (pages == null) { throw new ArgumentNullException(nameof(pages)); } List <PurgeFailureInfo> failedPages = null; // You can even purge pages from different sites. foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p))) { var site = sitePages.Key.Site; var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; using (site.BeginActionScope(sitePages, options)) { foreach (var partition in sitePages.Partition(titleLimit).Select(partition => partition.ToList())) { string titles; string ids; if (sitePages.Key.HasTitle) { // If a page has both title and ID information, // we will use title anyway. site.Logger.LogDebug("Purging {Count} pages by title.", partition.Count); titles = MediaWikiHelper.JoinValues(partition.Select(p => p.Title)); ids = null; } else { site.Logger.LogDebug("Purging {Count} pages by ID.", partition.Count); Debug.Assert(sitePages.All(p => p.PageStub.HasId)); titles = null; ids = MediaWikiHelper.JoinValues(partition.Select(p => p.Id)); } try { var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(new { action = "purge", titles = titles, pageids = ids, forcelinkupdate = (options & PagePurgeOptions.ForceLinkUpdate) == PagePurgeOptions.ForceLinkUpdate, forcerecursivelinkupdate = (options & PagePurgeOptions.ForceRecursiveLinkUpdate) == PagePurgeOptions.ForceRecursiveLinkUpdate, }), cancellationToken); // Now check whether the pages have been purged successfully. foreach (var jitem in jresult["purge"]) { if (jitem["missing"] != null || jitem["invalid"] != null) { if (failedPages == null) { failedPages = new List <PurgeFailureInfo>(); } failedPages.Add(new PurgeFailureInfo(MediaWikiHelper.PageStubFromJson((JObject)jitem), (string)jitem["invalidreason"])); } } } catch (OperationFailedException ex) { if (ex.ErrorCode == "cantpurge") { throw new UnauthorizedOperationException(ex); } throw; } } } } return(failedPages ?? emptyPurgeFailures); }
/// <summary> /// Refresh a sequence of pages. /// </summary> public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, IWikiPageQueryProvider options, CancellationToken cancellationToken) { if (pages == null) { throw new ArgumentNullException(nameof(pages)); } // You can even fetch pages from different sites. foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p))) { var site = sitePages.Key.Site; var queryParams = options.EnumParameters(site.SiteInfo.Version).ToDictionary(); var titleLimit = options.GetMaxPaginationSize(site.SiteInfo.Version, site.AccountInfo.HasRight(UserRights.ApiHighLimits)); using (site.BeginActionScope(sitePages, options)) { foreach (var partition in sitePages.Partition(titleLimit)) { if (sitePages.Key.HasTitle) { // If a page has both title and ID information, // we will use title anyway. site.Logger.LogDebug("Fetching {Count} pages by title.", partition.Count); queryParams["titles"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Title)); } else { site.Logger.LogDebug("Fetching {Count} pages by ID.", partition.Count); Debug.Assert(sitePages.All(p => p.PageStub.HasId)); queryParams["pageids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id)); } var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken); var jquery = (JObject)jobj["query"]; var continuationStatus = ParseContinuationParameters(jobj, queryParams, null); // Process continuation caused by props (e.g. langlinks) that contain a list that is too long. if (continuationStatus != CONTINUATION_DONE) { var queryParams1 = new Dictionary <string, object>(); var continuationParams = new Dictionary <string, object>(); var jobj1 = jobj; ParseContinuationParameters(jobj1, queryParams1, continuationParams); while (continuationStatus != CONTINUATION_DONE) { if (continuationStatus == CONTINUATION_LOOP) { throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop); } Debug.Assert(continuationStatus == CONTINUATION_AVAILABLE); site.Logger.LogDebug("Detected query continuation. PartitionCount={PartitionCount}.", partition.Count); queryParams1.Clear(); queryParams1.MergeFrom(queryParams); queryParams1.MergeFrom(continuationParams); jobj1 = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams1), cancellationToken); var jquery1 = jobj1["query"]; if (jquery1.HasValues) { // Merge JSON response jquery.Merge(jquery1); } continuationStatus = ParseContinuationParameters(jobj1, queryParams1, continuationParams); } } if (sitePages.Key.HasTitle) { // Process title normalization. var normalized = jquery["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); // Process redirects. var redirects = jquery["redirects"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); var pageInfoDict = ((JObject)jquery["pages"]).Properties() .ToDictionary(p => (string)p.Value["title"]); foreach (var page in partition) { var title = page.Title; // Normalize the title first. if (normalized?.ContainsKey(title) ?? false) { title = normalized[title]; } // Then process the redirects. var redirectTrace = new List <string>(); while (redirects?.ContainsKey(title) ?? false) { redirectTrace.Add(title); // Adds the last title var next = redirects[title]; if (redirectTrace.Contains(next)) { throw new InvalidOperationException(string.Format(Prompts.ExceptionWikiPageResolveCircularRedirect1, string.Join("->", redirectTrace))); } title = next; } // Finally, get the page. var pageInfo = pageInfoDict[title]; if (redirectTrace.Count > 0) { page.RedirectPath = redirectTrace; } MediaWikiHelper.PopulatePageFromJson(page, (JObject)pageInfo.Value, options); } } else { foreach (var page in partition) { var jPage = (JObject)jquery["pages"][page.Id.ToString(CultureInfo.InvariantCulture)]; MediaWikiHelper.PopulatePageFromJson(page, jPage, options); } } } } } }