/// <summary>
/// Construct a sequence of <see cref="WikiPageStub"/> from the given page IDs.
/// </summary>
/// <param name="site">The site in which to query for the pages.</param>
/// <param name="ids">The page IDs to query.</param>
/// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="ids"/> is <c>null</c>.</exception>
/// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns>
/// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks>
public static async IAsyncEnumerable <WikiPageStub> FromPageIds(WikiSite site, IEnumerable <int> ids,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
? 500
: 50;
foreach (var partition in ids.Partition(titleLimit))
{
var jresult = await site.InvokeMediaWikiApiAsync(
new MediaWikiFormRequestMessage(new { action = "query", pageids = MediaWikiHelper.JoinValues(partition), }), cancellationToken);
Debug.Assert(jresult["query"] != null);
var jpages = jresult["query"]["pages"];
using (ExecutionContextStash.Capture())
foreach (var id in partition)
{
var jpage = jpages[id.ToString(CultureInfo.InvariantCulture)];
if (jpage["missing"] == null)
{
yield return(new WikiPageStub(id, (string)jpage["title"], (int)jpage["ns"]));
}
else
{
yield return(new WikiPageStub(id, MissingPageTitle, UnknownNamespaceId));
}
}
}
}
/// <summary>
/// Asynchronously enumerates the topics in this board.
/// </summary>
/// <param name="options">Enumeration options.</param>
/// <param name="pageSize">
/// How many topics should be fetched in batch per MediaWiki API request.
/// No more than 100 (100 for bots) is allowed.
/// </param>
public async IAsyncEnumerable <Topic> EnumTopicsAsync(TopicListingOptions options, int pageSize,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
if (pageSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(pageSize));
}
var sortParam = "user";
if ((options & TopicListingOptions.OrderByPosted) == TopicListingOptions.OrderByPosted)
{
sortParam = "newest";
}
if ((options & TopicListingOptions.OrderByUpdated) == TopicListingOptions.OrderByUpdated)
{
sortParam = "updated";
}
var queryParams = new Dictionary <string, object>
{
{ "action", "flow" },
{ "submodule", "view-topiclist" },
{ "page", Title },
{ "vtlsortby", sortParam },
{ "vtlsavesortby", (options & TopicListingOptions.SaveSortingPreference) == TopicListingOptions.SaveSortingPreference },
{ "vtllimit", pageSize },
{ "vtlformat", "wikitext" },
};
NEXT_PAGE:
var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken);
var jtopiclist = (JObject)jresult["flow"]["view-topiclist"]["result"]["topiclist"];
using (ExecutionContextStash.Capture())
foreach (var t in Topic.FromJsonTopicList(Site, jtopiclist))
{
yield return(t);
}
// 2018-07-30 flow.view-topiclist.result.topiclist.links.pagination is [] instead of null for boards without pagination.
var jpagination = jtopiclist["links"]?["pagination"];
var nextPageUrl = jpagination == null || jpagination is JArray
? null
: (string)jpagination["fwd"]?["url"];
if (nextPageUrl != null)
{
var urlParams = FlowUtility.ParseUrlQueryParametrs(nextPageUrl);
foreach (var pa in urlParams)
{
if (pa.Key.StartsWith("topiclist_"))
{
queryParams["vtl" + pa.Key[10..]] = pa.Value;
/// <summary>
/// Construct a sequence of <see cref="WikiPageStub"/> from the given page titles.
/// </summary>
/// <param name="site">The site in which to query for the pages.</param>
/// <param name="titles">The page IDs to query.</param>
/// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="titles"/> is <c>null</c>.</exception>
/// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns>
/// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks>
public static async IAsyncEnumerable <WikiPageStub> FromPageTitles(WikiSite site, IEnumerable <string> titles,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
if (site == null)
{
throw new ArgumentNullException(nameof(site));
}
if (titles == null)
{
throw new ArgumentNullException(nameof(titles));
}
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
? 500
: 50;
foreach (var partition in titles.Partition(titleLimit))
{
var jresult = await site.InvokeMediaWikiApiAsync(
new MediaWikiFormRequestMessage(new { action = "query", titles = MediaWikiHelper.JoinValues(partition), }), cancellationToken);
Debug.Assert(jresult["query"] != null);
// Process title normalization.
var normalizedDict = jresult["query"]["normalized"]?.ToDictionary(n => (string)n["from"],
n => (string)n["to"]);
var pageDict = ((JObject)jresult["query"]["pages"]).Properties()
.ToDictionary(p => (string)p.Value["title"], p => p.Value);
using var ecs = ExecutionContextStash.Capture();
foreach (var name in partition)
{
if (normalizedDict == null || !normalizedDict.TryGetValue(name, out var normalizedName))
{
normalizedName = name;
}
var jpage = pageDict[normalizedName];
if (jpage["missing"] == null)
{
yield return(new WikiPageStub((int)jpage["pageid"], (string)jpage["title"], (int)jpage["ns"]));
}
else
{
yield return(new WikiPageStub(MissingPageIdMask, (string)jpage["title"], (int)jpage["ns"]));
}
}
}
}
/// <summary>
/// Asynchronously fetches the specified users' information.
/// </summary>
/// <param name="site">The site to issue the request.</param>
/// <param name="userNames">The user names to be fetched.</param>
/// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="userNames"/> is <c>null</c>.</exception>
/// <returns>
/// An asynchronous sequence containing the detailed user information.
/// The user names are normalized by the server. Inexistent user names are skipped.
/// </returns>
public static async IAsyncEnumerable <UserInfo> FetchUsersAsync(this WikiaSite site, IEnumerable <string> userNames,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
if (site == null)
{
throw new ArgumentNullException(nameof(site));
}
if (userNames == null)
{
throw new ArgumentNullException(nameof(userNames));
}
using (site.BeginActionScope(null, (object)(userNames as ICollection)))
{
foreach (var names in userNames.Partition(100))
{
JToken jresult;
try
{
jresult = await site.InvokeWikiaApiAsync("/User/Details",
new WikiaQueryRequestMessage(new { ids = string.Join(", ", names) }), cancellationToken);
}
catch (WikiaApiException ex) when(ex.ErrorType == "NotFoundApiException")
{
// All the usesers in this batch are not found.
// Pity.
continue;
}
var basePath = (string)jresult["basepath"];
var users = jresult["items"].ToObject <ICollection <UserInfo> >();
if (basePath != null)
{
foreach (var user in users)
{
user.ApplyBasePath(basePath);
}
}
using (ExecutionContextStash.Capture())
foreach (var user in users)
{
yield return(user);
}
}
}
}
public static async IAsyncEnumerable <Post> EnumArticleCommentsAsync(Board board, PostQueryOptions options,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
IList <Post> PostsFromJsonOutline(JObject commentList)
{
return(commentList.Properties().Select(p =>
{
var post = new Post(board.Site, board.Page, Convert.ToInt32(p.Name));
var level2 = p.Value["level2"];
if (level2 != null && level2.HasValues)
{
post.Replies = new ReadOnlyCollection <Post>(((JObject)level2).Properties().Select(p2 =>
new Post(board.Site, board.Page, Convert.ToInt32(p2.Name))).ToList());
}
return post;
}).ToList());
}
IEnumerable <Post> PostsAndDescendants(IEnumerable <Post> posts)
{
foreach (var p in posts)
{
yield return(p);
if (p.Replies.Count > 0)
{
foreach (var p2 in p.Replies)
{
yield return(p2);
// Wikia only supports level-2 comments for now.
Debug.Assert(p2.Replies.Count == 0);
}
}
}
}
using (board.Site.BeginActionScope(board))
{
// Refresh to get the page id.
if (!board.Page.HasId)
{
await board.RefreshAsync(cancellationToken);
}
if (!board.Exists)
{
yield break;
}
var pagesCount = 1;
for (int page = 1; page <= pagesCount; page++)
{
var jroot = await board.Site.InvokeNirvanaAsync(
new WikiaQueryRequestMessage(new
{
format = "json",
controller = "ArticleComments",
method = "Content",
articleId = board.Page.Id,
page = page
}), WikiaJsonResponseParser.Default, cancellationToken);
// Build comment structure.
var jcomments = jroot["commentListRaw"];
if (jcomments != null && jcomments.HasValues)
{
var comments = PostsFromJsonOutline((JObject)jcomments);
pagesCount = (int)jroot["pagesCount"];
await RefreshPostsAsync(PostsAndDescendants(comments), options, cancellationToken);
using (ExecutionContextStash.Capture())
foreach (var c in comments)
{
yield return(c);
}
}
}
}
}
/// <inheritdoc />
/// <exception cref="OperationFailedException">
/// (When enumerating) There is any MediaWiki API failure during the operation.
/// </exception>
/// <exception cref="Exception">
/// (When enumerating) There can be other types of errors thrown.
/// See the respective <see cref="OnEnumItemsFailed"/> override documentations in the implementation classes.
/// </exception>
public async IAsyncEnumerable <T> EnumItemsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var baseQueryParams = new Dictionary <string, object> {
{ "action", "query" }, { "maxlag", 5 }, { "list", ListName },
};
foreach (var p in EnumListParameters())
{
baseQueryParams.Add(p.Key, p.Value);
}
cancellationToken.ThrowIfCancellationRequested();
var continuationParams = new Dictionary <string, object>();
using var scope = Site.BeginActionScope(this);
// query parameters for this batch. The content/ref will be modified below.
var queryParams = new Dictionary <string, object>();
while (true)
{
queryParams.Clear();
queryParams.MergeFrom(baseQueryParams);
queryParams.MergeFrom(continuationParams);
JToken jresult;
JToken listNode;
try
{
jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken);
listNode = RequestHelper.FindQueryResponseItemsRoot(jresult, ListName);
}
catch (Exception ex)
{
OnEnumItemsFailed(ex);
throw;
}
if (listNode != null)
{
using (ExecutionContextStash.Capture())
{
foreach (var n in listNode)
{
yield return(ItemFromJson(n));
}
}
}
// Check for continuation.
switch (RequestHelper.ParseContinuationParameters(jresult, queryParams, continuationParams))
{
case RequestHelper.CONTINUATION_DONE:
yield break;
case RequestHelper.CONTINUATION_AVAILABLE:
if (listNode == null)
{
Site.Logger.LogWarning("Empty query page with continuation received.");
}
break;
case RequestHelper.CONTINUATION_LOOP:
Site.Logger.LogWarning("Continuation information provided by server response leads to infinite loop. {RawData}",
RequestHelper.FindQueryContinuationParameterRoot(jresult));
// The following is just last effort.
var outOfLoop = false;
if (CompatibilityOptions != null)
{
if ((CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.FetchMore) ==
WikiListContinuationLoopBehaviors.FetchMore)
{
// xxlimit (length = 7)
var limitParamName =
queryParams.Keys.FirstOrDefault(k => k.Length == 7 && k.EndsWith("limit", StringComparison.Ordinal));
if (limitParamName == null)
{
Site.Logger.LogWarning("Failed to find the underlying parameter name for PaginationSize.");
}
else
{
var maxLimit = Site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 1000 : 500;
var currentLimit = Math.Max(PaginationSize, 50);
// Continuously expand PaginationSize, hopefully we can retrieve some different continuation param value.
while (currentLimit < maxLimit)
{
currentLimit = Math.Min(maxLimit, currentLimit * 2);
Site.Logger.LogDebug("Try to fetch more with {ParamName}={ParamValue}.", limitParamName, currentLimit);
queryParams.Clear();
queryParams.MergeFrom(baseQueryParams);
queryParams.MergeFrom(continuationParams);
queryParams[limitParamName] = currentLimit;
var jresult2 = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams),
cancellationToken);
var applyResult = RequestHelper.ParseContinuationParameters(jresult2, queryParams, continuationParams);
switch (applyResult)
{
case RequestHelper.CONTINUATION_AVAILABLE:
case RequestHelper.CONTINUATION_DONE:
var listNode2 = RequestHelper.FindQueryResponseItemsRoot(jresult2, ListName);
Site.Logger.LogInformation("Successfully got out of the continuation loop.");
if (listNode2 != null)
{
if (listNode != null)
{
// Eliminate items that we have already yielded.
var yieldedItems = new HashSet <JToken>(listNode, new JTokenEqualityComparer());
using (ExecutionContextStash.Capture())
foreach (var n in listNode2.Where(n => !yieldedItems.Contains(n)))
{
yield return(ItemFromJson(n));
}
}
else
{
using (ExecutionContextStash.Capture())
foreach (var n in listNode2)
{
yield return(ItemFromJson(n));
}
}
}
outOfLoop = true;
if (applyResult == RequestHelper.CONTINUATION_DONE)
{
yield break;
}
break;
case RequestHelper.CONTINUATION_LOOP:
break;
}
}
}
}
//if (!outOfLoop && (CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.SkipItems) ==
// WikiListContinuationLoopBehaviors.SkipItems)
//{
//}
}
if (!outOfLoop)
{
throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop);
}
break;
}
}
}
