/// <summary>
/// Creates a list of <see cref="WikiPage"/> based on JSON query result.
/// </summary>
/// <param name="site">A <see cref="Site"/> object.</param>
/// <param name="queryNode">The <c>qurey</c> node value object of JSON result.</param>
/// <param name="options"></param>
/// <returns>Retrieved pages.</returns>
internal static IList <WikiPage> FromJsonQueryResult(WikiSite site, JObject queryNode, IWikiPageQueryProvider options)
{
if (site == null)
{
throw new ArgumentNullException(nameof(site));
}
if (queryNode == null)
{
throw new ArgumentNullException(nameof(queryNode));
}
var pages = (JObject)queryNode["pages"];
if (pages == null)
{
return(EmptyPages);
}
// If query.xxx.index exists, sort the pages by the given index.
// This is specifically used with SearchGenerator, to keep the search result in order.
// For other generators, this property simply does not exist.
// See https://www.mediawiki.org/wiki/API_talk:Query#On_the_order_of_titles_taken_out_of_generator .
return(pages.Properties().OrderBy(page => (int?)page.Value["index"])
.Select(page =>
{
var newInst = new WikiPage(site, 0);
MediaWikiHelper.PopulatePageFromJson(newInst, (JObject)page.Value, options);
return newInst;
}).ToList());
}
/// <summary>
/// Refresh a sequence of pages.
/// </summary>
public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, IWikiPageQueryProvider options, CancellationToken cancellationToken)
{
if (pages == null)
{
throw new ArgumentNullException(nameof(pages));
}
// You can even fetch pages from different sites.
foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p)))
{
var site = sitePages.Key.Site;
var queryParams = options.EnumParameters(site.SiteInfo.Version).ToDictionary();
var titleLimit = options.GetMaxPaginationSize(site.SiteInfo.Version, site.AccountInfo.HasRight(UserRights.ApiHighLimits));
using (site.BeginActionScope(sitePages, options))
{
foreach (var partition in sitePages.Partition(titleLimit))
{
if (sitePages.Key.HasTitle)
{
// If a page has both title and ID information,
// we will use title anyway.
site.Logger.LogDebug("Fetching {Count} pages by title.", partition.Count);
queryParams["titles"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Title));
}
else
{
site.Logger.LogDebug("Fetching {Count} pages by ID.", partition.Count);
Debug.Assert(sitePages.All(p => p.PageStub.HasId));
queryParams["pageids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id));
}
var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken);
var jquery = (JObject)jobj["query"];
var continuationStatus = ParseContinuationParameters(jobj, queryParams, null);
// Process continuation caused by props (e.g. langlinks) that contain a list that is too long.
if (continuationStatus != CONTINUATION_DONE)
{
var queryParams1 = new Dictionary <string, object>();
var continuationParams = new Dictionary <string, object>();
var jobj1 = jobj;
ParseContinuationParameters(jobj1, queryParams1, continuationParams);
while (continuationStatus != CONTINUATION_DONE)
{
if (continuationStatus == CONTINUATION_LOOP)
{
throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop);
}
Debug.Assert(continuationStatus == CONTINUATION_AVAILABLE);
site.Logger.LogDebug("Detected query continuation. PartitionCount={PartitionCount}.", partition.Count);
queryParams1.Clear();
queryParams1.MergeFrom(queryParams);
queryParams1.MergeFrom(continuationParams);
jobj1 = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams1), cancellationToken);
var jquery1 = jobj1["query"];
if (jquery1.HasValues)
{
// Merge JSON response
jquery.Merge(jquery1);
}
continuationStatus = ParseContinuationParameters(jobj1, queryParams1, continuationParams);
}
}
if (sitePages.Key.HasTitle)
{
// Process title normalization.
var normalized = jquery["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]);
// Process redirects.
var redirects = jquery["redirects"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]);
var pageInfoDict = ((JObject)jquery["pages"]).Properties()
.ToDictionary(p => (string)p.Value["title"]);
foreach (var page in partition)
{
var title = page.Title;
// Normalize the title first.
if (normalized?.ContainsKey(title) ?? false)
{
title = normalized[title];
}
// Then process the redirects.
var redirectTrace = new List <string>();
while (redirects?.ContainsKey(title) ?? false)
{
redirectTrace.Add(title); // Adds the last title
var next = redirects[title];
if (redirectTrace.Contains(next))
{
throw new InvalidOperationException(string.Format(Prompts.ExceptionWikiPageResolveCircularRedirect1, string.Join("->", redirectTrace)));
}
title = next;
}
// Finally, get the page.
var pageInfo = pageInfoDict[title];
if (redirectTrace.Count > 0)
{
page.RedirectPath = redirectTrace;
}
MediaWikiHelper.PopulatePageFromJson(page, (JObject)pageInfo.Value, options);
}
}
else
{
foreach (var page in partition)
{
var jPage = (JObject)jquery["pages"][page.Id.ToString(CultureInfo.InvariantCulture)];
MediaWikiHelper.PopulatePageFromJson(page, jPage, options);
}
}
}
}
}
}
/// <summary>
/// Refresh a sequence of pages.
/// </summary>
public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, IWikiPageQueryProvider options, CancellationToken cancellationToken)
{
if (pages == null)
{
throw new ArgumentNullException(nameof(pages));
}
// You can even fetch pages from different sites.
foreach (var sitePages in pages.GroupBy(p => new WikiPageGroupKey(p)))
{
var site = sitePages.Key.Site;
var queryParams = options.EnumParameters().ToDictionary();
var titleLimit = options.GetMaxPaginationSize(site.AccountInfo.HasRight(UserRights.ApiHighLimits));
using (site.BeginActionScope(sitePages, options))
{
foreach (var partition in sitePages.Partition(titleLimit))
{
if (sitePages.Key.HasTitle)
{
// If a page has both title and ID information,
// we will use title anyway.
site.Logger.LogDebug("Fetching {Count} pages by title.", partition.Count);
queryParams["titles"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Title));
}
else
{
site.Logger.LogDebug("Fetching {Count} pages by ID.", partition.Count);
Debug.Assert(sitePages.All(p => p.PageStub.HasId));
queryParams["pageids"] = MediaWikiHelper.JoinValues(partition.Select(p => p.Id));
}
// For single-page fetching, force fetching 1 revision only.
if (partition.Count == 1)
{
queryParams["rvlimit"] = 1;
}
else
{
queryParams.Remove("rvlimit");
}
var jobj = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken);
if (sitePages.Key.HasTitle)
{
// Process title normalization.
var normalized = jobj["query"]["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]);
// Process redirects.
var redirects = jobj["query"]["redirects"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]);
var pageInfoDict = ((JObject)jobj["query"]["pages"]).Properties()
.ToDictionary(p => (string)p.Value["title"]);
foreach (var page in partition)
{
var title = page.Title;
// Normalize the title first.
if (normalized?.ContainsKey(title) ?? false)
{
title = normalized[title];
}
// Then process the redirects.
var redirectTrace = new List <string>();
while (redirects?.ContainsKey(title) ?? false)
{
redirectTrace.Add(title); // Adds the last title
var next = redirects[title];
if (redirectTrace.Contains(next))
{
throw new InvalidOperationException($"Cannot resolve circular redirect: {string.Join("->", redirectTrace)}.");
}
title = next;
}
// Finally, get the page.
var pageInfo = pageInfoDict[title];
if (redirectTrace.Count > 0)
{
page.RedirectPath = redirectTrace;
}
MediaWikiHelper.PopulatePageFromJson(page, (JObject)pageInfo.Value, options);
}
}
else
{
foreach (var page in partition)
{
var jPage = (JObject)jobj["query"]["pages"][page.Id.ToString()];
MediaWikiHelper.PopulatePageFromJson(page, jPage, options);
}
}
}
}
}
}