/// <summary> /// Creates a list of <see cref="WikiPage"/> based on JSON query result. /// </summary> /// <param name="site">A <see cref="Site"/> object.</param> /// <param name="queryNode">The <c>qurey</c> node value object of JSON result.</param> /// <param name="options"></param> /// <returns>Retrieved pages.</returns> internal static IList <WikiPage> FromJsonQueryResult(WikiSite site, JObject queryNode, IWikiPageQueryProvider options) { if (site == null) { throw new ArgumentNullException(nameof(site)); } if (queryNode == null) { throw new ArgumentNullException(nameof(queryNode)); } var pages = (JObject)queryNode["pages"]; if (pages == null) { return(EmptyPages); } // If query.xxx.index exists, sort the pages by the given index. // This is specifically used with SearchGenerator, to keep the search result in order. // For other generators, this property simply does not exist. // See https://www.mediawiki.org/wiki/API_talk:Query#On_the_order_of_titles_taken_out_of_generator . return(pages.Properties().OrderBy(page => (int?)page.Value["index"]) .Select(page => { var newInst = new WikiPage(site, 0); MediaWikiHelper.PopulatePageFromJson(newInst, (JObject)page.Value, options); return newInst; }).ToList()); }
/// <summary> /// Refresh a sequence of pages. /// </summary> public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, IWikiPageQueryProvider options, CancellationToken cancellationToken) { if (pages == null) { throw new ArgumentNullException(nameof(pages)); } // You can even fetch pages from different sites. foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p))) { var site = sitePages.Key.Site; var queryParams = options.EnumParameters(site.SiteInfo.Version).ToDictionary(); var titleLimit = options.GetMaxPaginationSize(site.SiteInfo.Version, site.AccountInfo.HasRight(UserRights.ApiHighLimits)); using (site.BeginActionScope(sitePages, options)) { foreach (var partition in sitePages.Partition(titleLimit)) { if (sitePages.Key.HasTitle) { // If a page has both title and ID information, // we will use title anyway. site.Logger.LogDebug("Fetching {Count} pages by title.", partition.Count); queryParams["titles"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Title)); } else { site.Logger.LogDebug("Fetching {Count} pages by ID.", partition.Count); Debug.Assert(sitePages.All(p => p.PageStub.HasId)); queryParams["pageids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id)); } var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken); var jquery = (JObject)jobj["query"]; var continuationStatus = ParseContinuationParameters(jobj, queryParams, null); // Process continuation caused by props (e.g. langlinks) that contain a list that is too long. if (continuationStatus != CONTINUATION_DONE) { var queryParams1 = new Dictionary <string, object>(); var continuationParams = new Dictionary <string, object>(); var jobj1 = jobj; ParseContinuationParameters(jobj1, queryParams1, continuationParams); while (continuationStatus != CONTINUATION_DONE) { if (continuationStatus == CONTINUATION_LOOP) { throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop); } Debug.Assert(continuationStatus == CONTINUATION_AVAILABLE); site.Logger.LogDebug("Detected query continuation. PartitionCount={PartitionCount}.", partition.Count); queryParams1.Clear(); queryParams1.MergeFrom(queryParams); queryParams1.MergeFrom(continuationParams); jobj1 = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams1), cancellationToken); var jquery1 = jobj1["query"]; if (jquery1.HasValues) { // Merge JSON response jquery.Merge(jquery1); } continuationStatus = ParseContinuationParameters(jobj1, queryParams1, continuationParams); } } if (sitePages.Key.HasTitle) { // Process title normalization. var normalized = jquery["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); // Process redirects. var redirects = jquery["redirects"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); var pageInfoDict = ((JObject)jquery["pages"]).Properties() .ToDictionary(p => (string)p.Value["title"]); foreach (var page in partition) { var title = page.Title; // Normalize the title first. if (normalized?.ContainsKey(title) ?? false) { title = normalized[title]; } // Then process the redirects. var redirectTrace = new List <string>(); while (redirects?.ContainsKey(title) ?? false) { redirectTrace.Add(title); // Adds the last title var next = redirects[title]; if (redirectTrace.Contains(next)) { throw new InvalidOperationException(string.Format(Prompts.ExceptionWikiPageResolveCircularRedirect1, string.Join("->", redirectTrace))); } title = next; } // Finally, get the page. var pageInfo = pageInfoDict[title]; if (redirectTrace.Count > 0) { page.RedirectPath = redirectTrace; } MediaWikiHelper.PopulatePageFromJson(page, (JObject)pageInfo.Value, options); } } else { foreach (var page in partition) { var jPage = (JObject)jquery["pages"][page.Id.ToString(CultureInfo.InvariantCulture)]; MediaWikiHelper.PopulatePageFromJson(page, jPage, options); } } } } } }
/// <summary> /// Refresh a sequence of pages. /// </summary> public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, IWikiPageQueryProvider options, CancellationToken cancellationToken) { if (pages == null) { throw new ArgumentNullException(nameof(pages)); } // You can even fetch pages from different sites. foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p))) { var site = sitePages.Key.Site; var queryParams = options.EnumParameters().ToDictionary(); var titleLimit = options.GetMaxPaginationSize(site.AccountInfo.HasRight(UserRights.ApiHighLimits)); using (site.BeginActionScope(sitePages, options)) { foreach (var partition in sitePages.Partition(titleLimit)) { if (sitePages.Key.HasTitle) { // If a page has both title and ID information, // we will use title anyway. site.Logger.LogDebug("Fetching {Count} pages by title.", partition.Count); queryParams["titles"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Title)); } else { site.Logger.LogDebug("Fetching {Count} pages by ID.", partition.Count); Debug.Assert(sitePages.All(p => p.PageStub.HasId)); queryParams["pageids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id)); } // For single-page fetching, force fetching 1 revision only. if (partition.Count == 1) { queryParams["rvlimit"] = 1; } else { queryParams.Remove("rvlimit"); } var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken); if (sitePages.Key.HasTitle) { // Process title normalization. var normalized = jobj["query"]["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); // Process redirects. var redirects = jobj["query"]["redirects"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); var pageInfoDict = ((JObject)jobj["query"]["pages"]).Properties() .ToDictionary(p => (string)p.Value["title"]); foreach (var page in partition) { var title = page.Title; // Normalize the title first. if (normalized?.ContainsKey(title) ?? false) { title = normalized[title]; } // Then process the redirects. var redirectTrace = new List <string>(); while (redirects?.ContainsKey(title) ?? false) { redirectTrace.Add(title); // Adds the last title var next = redirects[title]; if (redirectTrace.Contains(next)) { throw new InvalidOperationException($"Cannot resolve circular redirect: {string.Join("->", redirectTrace)}."); } title = next; } // Finally, get the page. var pageInfo = pageInfoDict[title]; if (redirectTrace.Count > 0) { page.RedirectPath = redirectTrace; } MediaWikiHelper.PopulatePageFromJson(page, (JObject)pageInfo.Value, options); } } else { foreach (var page in partition) { var jPage = (JObject)jobj["query"]["pages"][page.Id.ToString()]; MediaWikiHelper.PopulatePageFromJson(page, jPage, options); } } } } } }