/// <summary> /// Asynchronously enumerates the topics in this board. /// </summary> /// <param name="options">Enumeration options.</param> /// <param name="pageSize"> /// How many topics should be fetched in batch per MediaWiki API request. /// No more than 100 (100 for bots) is allowed. /// </param> public IAsyncEnumerable <Topic> EnumTopicsAsync(TopicListingOptions options, int pageSize) { if (pageSize <= 0) { throw new ArgumentOutOfRangeException(nameof(pageSize)); } var sortParam = "user"; if ((options & TopicListingOptions.OrderByPosted) == TopicListingOptions.OrderByPosted) { sortParam = "newest"; } if ((options & TopicListingOptions.OrderByUpdated) == TopicListingOptions.OrderByUpdated) { sortParam = "updated"; } return(AsyncEnumerableFactory.FromAsyncGenerator <Topic>(async(sink, ct) => { var queryParams = new Dictionary <string, object> { { "action", "flow" }, { "submodule", "view-topiclist" }, { "page", Title }, { "vtlsortby", sortParam }, { "vtlsavesortby", (options & TopicListingOptions.SaveSortingPreference) == TopicListingOptions.SaveSortingPreference }, { "vtllimit", pageSize }, { "vtlformat", "wikitext" }, }; NEXT_PAGE: var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), ct); var jtopiclist = (JObject)jresult["flow"]["view-topiclist"]["result"]["topiclist"]; await sink.YieldAndWait(Topic.FromJsonTopicList(Site, jtopiclist)); // 2018-07-30 flow.view-topiclist.result.topiclist.links.pagination is [] instead of null for boards without pagination. var jpagination = jtopiclist["links"]?["pagination"]; var nextPageUrl = jpagination == null || jpagination is JArray ? null : (string)jpagination["fwd"]?["url"]; if (nextPageUrl != null) { var urlParams = FlowUtility.ParseUrlQueryParametrs(nextPageUrl); foreach (var pa in urlParams) { if (pa.Key.StartsWith("topiclist_")) { queryParams["vtl" + pa.Key.Substring(10)] = pa.Value; } } goto NEXT_PAGE; } })); }
public async void GeneratorExceptionTest() { async Task Generator(IAsyncEnumerableSink <int> sink) { await Task.Delay(100); await sink.YieldAndWait(10); await sink.YieldAndWait(20); throw new InvalidDataException(); } Assert.Equal(new[] { 10, 20 }, await AsyncEnumerableFactory.FromAsyncGenerator <int>(Generator).Take(2).ToArrayAsync()); await Assert.ThrowsAsync <InvalidDataException>(async() => await AsyncEnumerableFactory.FromAsyncGenerator <int>(Generator).ToArrayAsync()); }
public async void CancellationTest() { async Task Generator(IAsyncEnumerableSink <int> sink, CancellationToken token) { int value = 1; NEXT: value *= 2; await sink.YieldAndWait(value); await Task.Delay(100, token); goto NEXT; } var array = await AsyncEnumerableFactory.FromAsyncGenerator <int>(Generator).Take(5).ToArrayAsync(); Assert.Equal(new[] { 2, 4, 8, 16, 32 }, array); }
/// <summary> /// Asynchronously fetches the specified users' information. /// </summary> /// <param name="site">The site to issue the request.</param> /// <param name="userNames">The user names to be fetched.</param> /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="userNames"/> is <c>null</c>.</exception> /// <returns> /// An asynchronous sequence containing the detailed user information. /// The user names are normalized by the server. Inexistent user names are skipped. /// </returns> public static IAsyncEnumerable <UserInfo> FetchUsersAsync(this WikiaSite site, IEnumerable <string> userNames) { if (site == null) { throw new ArgumentNullException(nameof(site)); } if (userNames == null) { throw new ArgumentNullException(nameof(userNames)); } return(AsyncEnumerableFactory.FromAsyncGenerator <UserInfo>(async(sink, ct) => { using (site.BeginActionScope(null, (object)(userNames as ICollection))) { foreach (var names in userNames.Partition(100)) { JToken jresult; try { jresult = await site.InvokeWikiaApiAsync("/User/Details", new WikiaQueryRequestMessage(new { ids = string.Join(", ", names) }), ct); } catch (WikiaApiException ex) when(ex.ErrorType == "NotFoundApiException") { // All the usesers in this batch are not found. // Pity. continue; } var basePath = (string)jresult["basepath"]; var users = jresult["items"].ToObject <ICollection <UserInfo> >(); if (basePath != null) { foreach (var user in users) { user.ApplyBasePath(basePath); } } await sink.YieldAndWait(users); } } })); }
/// <inheritdoc /> public IAsyncEnumerable <LocalWikiSearchResultItem> EnumItemsAsync() { return(AsyncEnumerableFactory.FromAsyncGenerator <LocalWikiSearchResultItem>(async(sink, ct) => { int totalBatches = 1; for (int currentBatch = 1; currentBatch <= totalBatches; currentBatch++) { var jresult = await Site.InvokeWikiaApiAsync("/Search/List", new WikiaQueryRequestMessage(new { query = Keyword, type = "articles", rank = SerializeRank(RankingType), limit = PaginationSize, minArticleQuality = MinimumArticleQuality, namespaces = NamespaceIds == null ? null : string.Join(",", NamespaceIds), batch = currentBatch, }), ct); totalBatches = (int)jresult["batches"]; var items = jresult["items"].ToObject <IEnumerable <LocalWikiSearchResultItem> >( Utility.WikiaApiJsonSerializer); await sink.YieldAndWait(items); } })); }
/// <inheritdoc /> /// <exception cref="OperationFailedException"> /// (When enumerating) There is any MediaWiki API failure during the operation. /// </exception> /// <exception cref="Exception"> /// (When enumerating) There can be other types of errors thrown. /// See the respective <see cref="OnEnumItemsFailed"/> override documentations in the implementation classes. /// </exception> public IAsyncEnumerable <T> EnumItemsAsync() { return(AsyncEnumerableFactory.FromAsyncGenerator <T>(async(sink, ct) => { var baseQueryParams = new Dictionary <string, object> { { "action", "query" }, { "maxlag", 5 }, { "list", ListName }, }; foreach (var p in EnumListParameters()) { baseQueryParams.Add(p.Key, p.Value); } ct.ThrowIfCancellationRequested(); var continuationParams = new Dictionary <string, object>(); using (Site.BeginActionScope(this)) { // query parameters for this batch. The content/ref will be modified below. var queryParams = new Dictionary <string, object>(); while (true) { queryParams.Clear(); queryParams.MergeFrom(baseQueryParams); queryParams.MergeFrom(continuationParams); try { var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), ct); var listNode = RequestHelper.FindQueryResponseItemsRoot(jresult, ListName); if (listNode != null) { await sink.YieldAndWait(listNode.Select(ItemFromJson)); } // Check for continuation. switch (RequestHelper.ParseContinuationParameters(jresult, queryParams, continuationParams)) { case RequestHelper.CONTINUATION_DONE: return; case RequestHelper.CONTINUATION_AVAILABLE: if (listNode == null) { Site.Logger.LogWarning("Empty query page with continuation received."); } break; case RequestHelper.CONTINUATION_LOOP: Site.Logger.LogWarning("Continuation information provided by server response leads to infinite loop. {RawData}", RequestHelper.FindQueryContinuationParameterRoot(jresult)); // The following is just last effort. var outOfLoop = false; if (CompatibilityOptions != null) { if ((CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.FetchMore) == WikiListContinuationLoopBehaviors.FetchMore) { // xxlimit (length = 7) var limitParamName = queryParams.Keys.FirstOrDefault(k => k.Length == 7 && k.EndsWith("limit", StringComparison.Ordinal)); if (limitParamName == null) { Site.Logger.LogWarning("Failed to find the underlying parameter name for PaginationSize."); } else { var maxLimit = Site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 1000 : 500; var currentLimit = Math.Max(PaginationSize, 50); // Continuously expand PaginationSize, hopefully we can retrieve some different continuation param value. while (currentLimit < maxLimit) { currentLimit = Math.Min(maxLimit, currentLimit * 2); Site.Logger.LogDebug("Try to fetch more with {ParamName}={ParamValue}.", limitParamName, currentLimit); queryParams.Clear(); queryParams.MergeFrom(baseQueryParams); queryParams.MergeFrom(continuationParams); queryParams[limitParamName] = currentLimit; var jresult2 = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), ct); var applyResult = RequestHelper.ParseContinuationParameters(jresult2, queryParams, continuationParams); switch (applyResult) { case RequestHelper.CONTINUATION_AVAILABLE: case RequestHelper.CONTINUATION_DONE: var listNode2 = RequestHelper.FindQueryResponseItemsRoot(jresult2, ListName); Site.Logger.LogInformation("Successfully got out of the continuation loop."); if (listNode2 != null) { if (listNode != null) { // Eliminate items that we have already yielded. var yieldedItems = new HashSet <JToken>(listNode, new JTokenEqualityComparer()); await sink.YieldAndWait( listNode2.Where(n => !yieldedItems.Contains(n)).Select(ItemFromJson)); } else { await sink.YieldAndWait(listNode2.Select(ItemFromJson)); } } outOfLoop = true; if (applyResult == RequestHelper.CONTINUATION_DONE) { return; } break; case RequestHelper.CONTINUATION_LOOP: break; } } } } //if (!outOfLoop && (CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.SkipItems) == // WikiListContinuationLoopBehaviors.SkipItems) //{ //} } if (!outOfLoop) { throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop); } break; } } catch (Exception ex) { OnEnumItemsFailed(ex); throw; } } } })); }
public static IAsyncEnumerable <JObject> QueryWithContinuation(WikiSite site, IEnumerable <KeyValuePair <string, object> > parameters, Func <IDisposable> beginActionScope, bool distinctPages = false) { return(AsyncEnumerableFactory.FromAsyncGenerator <JObject>(async(sink, ct) => { ct.ThrowIfCancellationRequested(); var retrivedPageIds = distinctPages ? new HashSet <int>() : null; using (beginActionScope?.Invoke()) { var baseQueryParams = parameters.ToDictionary(p => p.Key, p => p.Value); Debug.Assert("query".Equals(baseQueryParams["action"])); var continuationParams = new Dictionary <string, object>(); while (true) { var queryParams = new Dictionary <string, object>(baseQueryParams); queryParams.MergeFrom(continuationParams); var jresult = await site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), ct); var jpages = (JObject)FindQueryResponseItemsRoot(jresult, "pages"); if (jpages != null) { if (retrivedPageIds != null) { // Remove duplicate results var duplicateKeys = new List <string>(jpages.Count); foreach (var jpage in jpages) { if (!retrivedPageIds.Add(Convert.ToInt32(jpage.Key))) { // The page has been retrieved before. duplicateKeys.Add(jpage.Key); } } var originalPageCount = jpages.Count; foreach (var k in duplicateKeys) { jpages.Remove(k); } if (originalPageCount != jpages.Count) { site.Logger.LogWarning( "Received {Count} results on {Site}, {DistinctCount} distinct results.", originalPageCount, site, jpages.Count); } } await sink.YieldAndWait(jpages); } switch (ParseContinuationParameters(jresult, queryParams, continuationParams)) { case CONTINUATION_DONE: return; case CONTINUATION_AVAILABLE: if (jpages == null) { site.Logger.LogWarning("Empty query page with continuation received on {Site}.", site); } // Continue the loop and fetch for the next page of query. break; case CONTINUATION_LOOP: throw new UnexpectedDataException(); } } } })); }
public static IAsyncEnumerable <Post> EnumArticleCommentsAsync(Board board, PostQueryOptions options) { IList <Post> PostsFromJsonOutline(JObject commentList) { return(commentList.Properties().Select(p => { var post = new Post(board.Site, board.Page, Convert.ToInt32(p.Name)); var level2 = p.Value["level2"]; if (level2 != null && level2.HasValues) { post.Replies = new ReadOnlyCollection <Post>(((JObject)level2).Properties().Select(p2 => new Post(board.Site, board.Page, Convert.ToInt32(p2.Name))).ToList()); } return post; }).ToList()); } IEnumerable <Post> PostsAndDescendants(IEnumerable <Post> posts) { foreach (var p in posts) { yield return(p); if (p.Replies.Count > 0) { foreach (var p2 in p.Replies) { yield return(p2); // Wikia only supports level-2 comments for now. Debug.Assert(p2.Replies.Count == 0); } } } } return(AsyncEnumerableFactory.FromAsyncGenerator <Post>(async(sink, ct) => { using (board.Site.BeginActionScope(board)) { // Refresh to get the page id. if (!board.Page.HasId) { await board.RefreshAsync(ct); } if (!board.Exists) { return; } var pagesCount = 1; for (int page = 1; page <= pagesCount; page++) { var jroot = await board.Site.InvokeNirvanaAsync(new WikiaQueryRequestMessage(new { format = "json", controller = "ArticleComments", method = "Content", articleId = board.Page.Id, page = page }), WikiaJsonResonseParser.Default, ct); // Build comment structure. var jcomments = jroot["commentListRaw"]; if (jcomments != null && jcomments.HasValues) { var comments = PostsFromJsonOutline((JObject)jcomments); pagesCount = (int)jroot["pagesCount"]; await RefreshPostsAsync(PostsAndDescendants(comments), options, ct); await sink.YieldAndWait(comments); } } } })); }