/// <summary> /// Construct a sequence of <see cref="WikiPageStub"/> from the given page IDs. /// </summary> /// <param name="site">The site in which to query for the pages.</param> /// <param name="ids">The page IDs to query.</param> /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="ids"/> is <c>null</c>.</exception> /// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns> /// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks> public static async IAsyncEnumerable <WikiPageStub> FromPageIds(WikiSite site, IEnumerable <int> ids, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; foreach (var partition in ids.Partition(titleLimit)) { var jresult = await site.InvokeMediaWikiApiAsync( new MediaWikiFormRequestMessage(new { action = "query", pageids = MediaWikiHelper.JoinValues(partition), }), cancellationToken); Debug.Assert(jresult["query"] != null); var jpages = jresult["query"]["pages"]; using (ExecutionContextStash.Capture()) foreach (var id in partition) { var jpage = jpages[id.ToString(CultureInfo.InvariantCulture)]; if (jpage["missing"] == null) { yield return(new WikiPageStub(id, (string)jpage["title"], (int)jpage["ns"])); } else { yield return(new WikiPageStub(id, MissingPageTitle, UnknownNamespaceId)); } } } }
/// <summary> /// Asynchronously enumerates the topics in this board. /// </summary> /// <param name="options">Enumeration options.</param> /// <param name="pageSize"> /// How many topics should be fetched in batch per MediaWiki API request. /// No more than 100 (100 for bots) is allowed. /// </param> public async IAsyncEnumerable <Topic> EnumTopicsAsync(TopicListingOptions options, int pageSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) { if (pageSize <= 0) { throw new ArgumentOutOfRangeException(nameof(pageSize)); } var sortParam = "user"; if ((options & TopicListingOptions.OrderByPosted) == TopicListingOptions.OrderByPosted) { sortParam = "newest"; } if ((options & TopicListingOptions.OrderByUpdated) == TopicListingOptions.OrderByUpdated) { sortParam = "updated"; } var queryParams = new Dictionary <string, object> { { "action", "flow" }, { "submodule", "view-topiclist" }, { "page", Title }, { "vtlsortby", sortParam }, { "vtlsavesortby", (options & TopicListingOptions.SaveSortingPreference) == TopicListingOptions.SaveSortingPreference }, { "vtllimit", pageSize }, { "vtlformat", "wikitext" }, }; NEXT_PAGE: var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken); var jtopiclist = (JObject)jresult["flow"]["view-topiclist"]["result"]["topiclist"]; using (ExecutionContextStash.Capture()) foreach (var t in Topic.FromJsonTopicList(Site, jtopiclist)) { yield return(t); } // 2018-07-30 flow.view-topiclist.result.topiclist.links.pagination is [] instead of null for boards without pagination. var jpagination = jtopiclist["links"]?["pagination"]; var nextPageUrl = jpagination == null || jpagination is JArray ? null : (string)jpagination["fwd"]?["url"]; if (nextPageUrl != null) { var urlParams = FlowUtility.ParseUrlQueryParametrs(nextPageUrl); foreach (var pa in urlParams) { if (pa.Key.StartsWith("topiclist_")) { queryParams["vtl" + pa.Key[10..]] = pa.Value;
/// <summary> /// Construct a sequence of <see cref="WikiPageStub"/> from the given page titles. /// </summary> /// <param name="site">The site in which to query for the pages.</param> /// <param name="titles">The page IDs to query.</param> /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="titles"/> is <c>null</c>.</exception> /// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns> /// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks> public static async IAsyncEnumerable <WikiPageStub> FromPageTitles(WikiSite site, IEnumerable <string> titles, [EnumeratorCancellation] CancellationToken cancellationToken = default) { if (site == null) { throw new ArgumentNullException(nameof(site)); } if (titles == null) { throw new ArgumentNullException(nameof(titles)); } var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; foreach (var partition in titles.Partition(titleLimit)) { var jresult = await site.InvokeMediaWikiApiAsync( new MediaWikiFormRequestMessage(new { action = "query", titles = MediaWikiHelper.JoinValues(partition), }), cancellationToken); Debug.Assert(jresult["query"] != null); // Process title normalization. var normalizedDict = jresult["query"]["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); var pageDict = ((JObject)jresult["query"]["pages"]).Properties() .ToDictionary(p => (string)p.Value["title"], p => p.Value); using var ecs = ExecutionContextStash.Capture(); foreach (var name in partition) { if (normalizedDict == null || !normalizedDict.TryGetValue(name, out var normalizedName)) { normalizedName = name; } var jpage = pageDict[normalizedName]; if (jpage["missing"] == null) { yield return(new WikiPageStub((int)jpage["pageid"], (string)jpage["title"], (int)jpage["ns"])); } else { yield return(new WikiPageStub(MissingPageIdMask, (string)jpage["title"], (int)jpage["ns"])); } } } }
/// <summary> /// Asynchronously fetches the specified users' information. /// </summary> /// <param name="site">The site to issue the request.</param> /// <param name="userNames">The user names to be fetched.</param> /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="userNames"/> is <c>null</c>.</exception> /// <returns> /// An asynchronous sequence containing the detailed user information. /// The user names are normalized by the server. Inexistent user names are skipped. /// </returns> public static async IAsyncEnumerable <UserInfo> FetchUsersAsync(this WikiaSite site, IEnumerable <string> userNames, [EnumeratorCancellation] CancellationToken cancellationToken = default) { if (site == null) { throw new ArgumentNullException(nameof(site)); } if (userNames == null) { throw new ArgumentNullException(nameof(userNames)); } using (site.BeginActionScope(null, (object)(userNames as ICollection))) { foreach (var names in userNames.Partition(100)) { JToken jresult; try { jresult = await site.InvokeWikiaApiAsync("/User/Details", new WikiaQueryRequestMessage(new { ids = string.Join(", ", names) }), cancellationToken); } catch (WikiaApiException ex) when(ex.ErrorType == "NotFoundApiException") { // All the usesers in this batch are not found. // Pity. continue; } var basePath = (string)jresult["basepath"]; var users = jresult["items"].ToObject <ICollection <UserInfo> >(); if (basePath != null) { foreach (var user in users) { user.ApplyBasePath(basePath); } } using (ExecutionContextStash.Capture()) foreach (var user in users) { yield return(user); } } } }
public static async IAsyncEnumerable <Post> EnumArticleCommentsAsync(Board board, PostQueryOptions options, [EnumeratorCancellation] CancellationToken cancellationToken = default) { IList <Post> PostsFromJsonOutline(JObject commentList) { return(commentList.Properties().Select(p => { var post = new Post(board.Site, board.Page, Convert.ToInt32(p.Name)); var level2 = p.Value["level2"]; if (level2 != null && level2.HasValues) { post.Replies = new ReadOnlyCollection <Post>(((JObject)level2).Properties().Select(p2 => new Post(board.Site, board.Page, Convert.ToInt32(p2.Name))).ToList()); } return post; }).ToList()); } IEnumerable <Post> PostsAndDescendants(IEnumerable <Post> posts) { foreach (var p in posts) { yield return(p); if (p.Replies.Count > 0) { foreach (var p2 in p.Replies) { yield return(p2); // Wikia only supports level-2 comments for now. Debug.Assert(p2.Replies.Count == 0); } } } } using (board.Site.BeginActionScope(board)) { // Refresh to get the page id. if (!board.Page.HasId) { await board.RefreshAsync(cancellationToken); } if (!board.Exists) { yield break; } var pagesCount = 1; for (int page = 1; page <= pagesCount; page++) { var jroot = await board.Site.InvokeNirvanaAsync( new WikiaQueryRequestMessage(new { format = "json", controller = "ArticleComments", method = "Content", articleId = board.Page.Id, page = page }), WikiaJsonResponseParser.Default, cancellationToken); // Build comment structure. var jcomments = jroot["commentListRaw"]; if (jcomments != null && jcomments.HasValues) { var comments = PostsFromJsonOutline((JObject)jcomments); pagesCount = (int)jroot["pagesCount"]; await RefreshPostsAsync(PostsAndDescendants(comments), options, cancellationToken); using (ExecutionContextStash.Capture()) foreach (var c in comments) { yield return(c); } } } } }
/// <inheritdoc /> /// <exception cref="OperationFailedException"> /// (When enumerating) There is any MediaWiki API failure during the operation. /// </exception> /// <exception cref="Exception"> /// (When enumerating) There can be other types of errors thrown. /// See the respective <see cref="OnEnumItemsFailed"/> override documentations in the implementation classes. /// </exception> public async IAsyncEnumerable <T> EnumItemsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { var baseQueryParams = new Dictionary <string, object> { { "action", "query" }, { "maxlag", 5 }, { "list", ListName }, }; foreach (var p in EnumListParameters()) { baseQueryParams.Add(p.Key, p.Value); } cancellationToken.ThrowIfCancellationRequested(); var continuationParams = new Dictionary <string, object>(); using var scope = Site.BeginActionScope(this); // query parameters for this batch. The content/ref will be modified below. var queryParams = new Dictionary <string, object>(); while (true) { queryParams.Clear(); queryParams.MergeFrom(baseQueryParams); queryParams.MergeFrom(continuationParams); JToken jresult; JToken listNode; try { jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken); listNode = RequestHelper.FindQueryResponseItemsRoot(jresult, ListName); } catch (Exception ex) { OnEnumItemsFailed(ex); throw; } if (listNode != null) { using (ExecutionContextStash.Capture()) { foreach (var n in listNode) { yield return(ItemFromJson(n)); } } } // Check for continuation. switch (RequestHelper.ParseContinuationParameters(jresult, queryParams, continuationParams)) { case RequestHelper.CONTINUATION_DONE: yield break; case RequestHelper.CONTINUATION_AVAILABLE: if (listNode == null) { Site.Logger.LogWarning("Empty query page with continuation received."); } break; case RequestHelper.CONTINUATION_LOOP: Site.Logger.LogWarning("Continuation information provided by server response leads to infinite loop. {RawData}", RequestHelper.FindQueryContinuationParameterRoot(jresult)); // The following is just last effort. var outOfLoop = false; if (CompatibilityOptions != null) { if ((CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.FetchMore) == WikiListContinuationLoopBehaviors.FetchMore) { // xxlimit (length = 7) var limitParamName = queryParams.Keys.FirstOrDefault(k => k.Length == 7 && k.EndsWith("limit", StringComparison.Ordinal)); if (limitParamName == null) { Site.Logger.LogWarning("Failed to find the underlying parameter name for PaginationSize."); } else { var maxLimit = Site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 1000 : 500; var currentLimit = Math.Max(PaginationSize, 50); // Continuously expand PaginationSize, hopefully we can retrieve some different continuation param value. while (currentLimit < maxLimit) { currentLimit = Math.Min(maxLimit, currentLimit * 2); Site.Logger.LogDebug("Try to fetch more with {ParamName}={ParamValue}.", limitParamName, currentLimit); queryParams.Clear(); queryParams.MergeFrom(baseQueryParams); queryParams.MergeFrom(continuationParams); queryParams[limitParamName] = currentLimit; var jresult2 = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken); var applyResult = RequestHelper.ParseContinuationParameters(jresult2, queryParams, continuationParams); switch (applyResult) { case RequestHelper.CONTINUATION_AVAILABLE: case RequestHelper.CONTINUATION_DONE: var listNode2 = RequestHelper.FindQueryResponseItemsRoot(jresult2, ListName); Site.Logger.LogInformation("Successfully got out of the continuation loop."); if (listNode2 != null) { if (listNode != null) { // Eliminate items that we have already yielded. var yieldedItems = new HashSet <JToken>(listNode, new JTokenEqualityComparer()); using (ExecutionContextStash.Capture()) foreach (var n in listNode2.Where(n => !yieldedItems.Contains(n))) { yield return(ItemFromJson(n)); } } else { using (ExecutionContextStash.Capture()) foreach (var n in listNode2) { yield return(ItemFromJson(n)); } } } outOfLoop = true; if (applyResult == RequestHelper.CONTINUATION_DONE) { yield break; } break; case RequestHelper.CONTINUATION_LOOP: break; } } } } //if (!outOfLoop && (CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.SkipItems) == // WikiListContinuationLoopBehaviors.SkipItems) //{ //} } if (!outOfLoop) { throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop); } break; } } }