Ejemplo n.º 1
0
        /// <summary>
        /// Construct a sequence of <see cref="WikiPageStub"/> from the given page IDs.
        /// </summary>
        /// <param name="site">The site in which to query for the pages.</param>
        /// <param name="ids">The page IDs to query.</param>
        /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="ids"/> is <c>null</c>.</exception>
        /// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns>
        /// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks>
        public static async IAsyncEnumerable <WikiPageStub> FromPageIds(WikiSite site, IEnumerable <int> ids,
                                                                        [EnumeratorCancellation] CancellationToken cancellationToken = default)
        {
            var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
                ? 500
                : 50;

            foreach (var partition in ids.Partition(titleLimit))
            {
                var jresult = await site.InvokeMediaWikiApiAsync(
                    new MediaWikiFormRequestMessage(new { action = "query", pageids = MediaWikiHelper.JoinValues(partition), }), cancellationToken);

                Debug.Assert(jresult["query"] != null);
                var jpages = jresult["query"]["pages"];
                using (ExecutionContextStash.Capture())
                    foreach (var id in partition)
                    {
                        var jpage = jpages[id.ToString(CultureInfo.InvariantCulture)];
                        if (jpage["missing"] == null)
                        {
                            yield return(new WikiPageStub(id, (string)jpage["title"], (int)jpage["ns"]));
                        }
                        else
                        {
                            yield return(new WikiPageStub(id, MissingPageTitle, UnknownNamespaceId));
                        }
                    }
            }
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Asynchronously enumerates the topics in this board.
        /// </summary>
        /// <param name="options">Enumeration options.</param>
        /// <param name="pageSize">
        /// How many topics should be fetched in batch per MediaWiki API request.
        /// No more than 100 (100 for bots) is allowed.
        /// </param>
        public async IAsyncEnumerable <Topic> EnumTopicsAsync(TopicListingOptions options, int pageSize,
                                                              [EnumeratorCancellation] CancellationToken cancellationToken = default)
        {
            if (pageSize <= 0)
            {
                throw new ArgumentOutOfRangeException(nameof(pageSize));
            }
            var sortParam = "user";

            if ((options & TopicListingOptions.OrderByPosted) == TopicListingOptions.OrderByPosted)
            {
                sortParam = "newest";
            }
            if ((options & TopicListingOptions.OrderByUpdated) == TopicListingOptions.OrderByUpdated)
            {
                sortParam = "updated";
            }
            var queryParams = new Dictionary <string, object>
            {
                { "action", "flow" },
                { "submodule", "view-topiclist" },
                { "page", Title },
                { "vtlsortby", sortParam },
                { "vtlsavesortby", (options & TopicListingOptions.SaveSortingPreference) == TopicListingOptions.SaveSortingPreference },
                { "vtllimit", pageSize },
                { "vtlformat", "wikitext" },
            };

NEXT_PAGE:
            var jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken);

            var jtopiclist = (JObject)jresult["flow"]["view-topiclist"]["result"]["topiclist"];

            using (ExecutionContextStash.Capture())
                foreach (var t in Topic.FromJsonTopicList(Site, jtopiclist))
                {
                    yield return(t);
                }
            // 2018-07-30 flow.view-topiclist.result.topiclist.links.pagination is [] instead of null for boards without pagination.
            var jpagination = jtopiclist["links"]?["pagination"];
            var nextPageUrl = jpagination == null || jpagination is JArray
                ? null
                : (string)jpagination["fwd"]?["url"];

            if (nextPageUrl != null)
            {
                var urlParams = FlowUtility.ParseUrlQueryParametrs(nextPageUrl);
                foreach (var pa in urlParams)
                {
                    if (pa.Key.StartsWith("topiclist_"))
                    {
                        queryParams["vtl" + pa.Key[10..]] = pa.Value;
Ejemplo n.º 3
0
        /// <summary>
        /// Construct a sequence of <see cref="WikiPageStub"/> from the given page titles.
        /// </summary>
        /// <param name="site">The site in which to query for the pages.</param>
        /// <param name="titles">The page IDs to query.</param>
        /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="titles"/> is <c>null</c>.</exception>
        /// <returns>A sequence of <see cref="WikiPageStub"/> containing the page information.</returns>
        /// <remarks>For how the missing pages are handled, see the "remarks" section of <see cref="WikiPage"/>.</remarks>
        public static async IAsyncEnumerable <WikiPageStub> FromPageTitles(WikiSite site, IEnumerable <string> titles,
                                                                           [EnumeratorCancellation] CancellationToken cancellationToken = default)
        {
            if (site == null)
            {
                throw new ArgumentNullException(nameof(site));
            }
            if (titles == null)
            {
                throw new ArgumentNullException(nameof(titles));
            }
            var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
                ? 500
                : 50;

            foreach (var partition in titles.Partition(titleLimit))
            {
                var jresult = await site.InvokeMediaWikiApiAsync(
                    new MediaWikiFormRequestMessage(new { action = "query", titles = MediaWikiHelper.JoinValues(partition), }), cancellationToken);

                Debug.Assert(jresult["query"] != null);
                // Process title normalization.
                var normalizedDict = jresult["query"]["normalized"]?.ToDictionary(n => (string)n["from"],
                                                                                  n => (string)n["to"]);
                var pageDict = ((JObject)jresult["query"]["pages"]).Properties()
                               .ToDictionary(p => (string)p.Value["title"], p => p.Value);
                using var ecs = ExecutionContextStash.Capture();
                foreach (var name in partition)
                {
                    if (normalizedDict == null || !normalizedDict.TryGetValue(name, out var normalizedName))
                    {
                        normalizedName = name;
                    }
                    var jpage = pageDict[normalizedName];
                    if (jpage["missing"] == null)
                    {
                        yield return(new WikiPageStub((int)jpage["pageid"], (string)jpage["title"], (int)jpage["ns"]));
                    }
                    else
                    {
                        yield return(new WikiPageStub(MissingPageIdMask, (string)jpage["title"], (int)jpage["ns"]));
                    }
                }
            }
        }
Ejemplo n.º 4
0
 /// <summary>
 /// Asynchronously fetches the specified users' information.
 /// </summary>
 /// <param name="site">The site to issue the request.</param>
 /// <param name="userNames">The user names to be fetched.</param>
 /// <exception cref="ArgumentNullException">Either <paramref name="site"/> or <paramref name="userNames"/> is <c>null</c>.</exception>
 /// <returns>
 /// An asynchronous sequence containing the detailed user information.
 /// The user names are normalized by the server. Inexistent user names are skipped.
 /// </returns>
 public static async IAsyncEnumerable <UserInfo> FetchUsersAsync(this WikiaSite site, IEnumerable <string> userNames,
                                                                 [EnumeratorCancellation] CancellationToken cancellationToken = default)
 {
     if (site == null)
     {
         throw new ArgumentNullException(nameof(site));
     }
     if (userNames == null)
     {
         throw new ArgumentNullException(nameof(userNames));
     }
     using (site.BeginActionScope(null, (object)(userNames as ICollection)))
     {
         foreach (var names in userNames.Partition(100))
         {
             JToken jresult;
             try
             {
                 jresult = await site.InvokeWikiaApiAsync("/User/Details",
                                                          new WikiaQueryRequestMessage(new { ids = string.Join(", ", names) }), cancellationToken);
             }
             catch (WikiaApiException ex) when(ex.ErrorType == "NotFoundApiException")
             {
                 // All the usesers in this batch are not found.
                 // Pity.
                 continue;
             }
             var basePath = (string)jresult["basepath"];
             var users    = jresult["items"].ToObject <ICollection <UserInfo> >();
             if (basePath != null)
             {
                 foreach (var user in users)
                 {
                     user.ApplyBasePath(basePath);
                 }
             }
             using (ExecutionContextStash.Capture())
                 foreach (var user in users)
                 {
                     yield return(user);
                 }
         }
     }
 }
Ejemplo n.º 5
0
        public static async IAsyncEnumerable <Post> EnumArticleCommentsAsync(Board board, PostQueryOptions options,
                                                                             [EnumeratorCancellation] CancellationToken cancellationToken = default)
        {
            IList <Post> PostsFromJsonOutline(JObject commentList)
            {
                return(commentList.Properties().Select(p =>
                {
                    var post = new Post(board.Site, board.Page, Convert.ToInt32(p.Name));
                    var level2 = p.Value["level2"];
                    if (level2 != null && level2.HasValues)
                    {
                        post.Replies = new ReadOnlyCollection <Post>(((JObject)level2).Properties().Select(p2 =>
                                                                                                           new Post(board.Site, board.Page, Convert.ToInt32(p2.Name))).ToList());
                    }
                    return post;
                }).ToList());
            }

            IEnumerable <Post> PostsAndDescendants(IEnumerable <Post> posts)
            {
                foreach (var p in posts)
                {
                    yield return(p);

                    if (p.Replies.Count > 0)
                    {
                        foreach (var p2 in p.Replies)
                        {
                            yield return(p2);

                            // Wikia only supports level-2 comments for now.
                            Debug.Assert(p2.Replies.Count == 0);
                        }
                    }
                }
            }

            using (board.Site.BeginActionScope(board))
            {
                // Refresh to get the page id.
                if (!board.Page.HasId)
                {
                    await board.RefreshAsync(cancellationToken);
                }
                if (!board.Exists)
                {
                    yield break;
                }
                var pagesCount = 1;
                for (int page = 1; page <= pagesCount; page++)
                {
                    var jroot = await board.Site.InvokeNirvanaAsync(
                        new WikiaQueryRequestMessage(new
                    {
                        format = "json",
                        controller = "ArticleComments",
                        method = "Content",
                        articleId = board.Page.Id,
                        page = page
                    }), WikiaJsonResponseParser.Default, cancellationToken);

                    // Build comment structure.
                    var jcomments = jroot["commentListRaw"];
                    if (jcomments != null && jcomments.HasValues)
                    {
                        var comments = PostsFromJsonOutline((JObject)jcomments);
                        pagesCount = (int)jroot["pagesCount"];
                        await RefreshPostsAsync(PostsAndDescendants(comments), options, cancellationToken);

                        using (ExecutionContextStash.Capture())
                            foreach (var c in comments)
                            {
                                yield return(c);
                            }
                    }
                }
            }
        }
Ejemplo n.º 6
0
        /// <inheritdoc />
        /// <exception cref="OperationFailedException">
        /// (When enumerating) There is any MediaWiki API failure during the operation.
        /// </exception>
        /// <exception cref="Exception">
        /// (When enumerating) There can be other types of errors thrown.
        /// See the respective <see cref="OnEnumItemsFailed"/> override documentations in the implementation classes.
        /// </exception>
        public async IAsyncEnumerable <T> EnumItemsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
        {
            var baseQueryParams = new Dictionary <string, object> {
                { "action", "query" }, { "maxlag", 5 }, { "list", ListName },
            };

            foreach (var p in EnumListParameters())
            {
                baseQueryParams.Add(p.Key, p.Value);
            }
            cancellationToken.ThrowIfCancellationRequested();
            var continuationParams = new Dictionary <string, object>();

            using var scope = Site.BeginActionScope(this);
            // query parameters for this batch. The content/ref will be modified below.
            var queryParams = new Dictionary <string, object>();

            while (true)
            {
                queryParams.Clear();
                queryParams.MergeFrom(baseQueryParams);
                queryParams.MergeFrom(continuationParams);
                JToken jresult;
                JToken listNode;
                try
                {
                    jresult = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams), cancellationToken);

                    listNode = RequestHelper.FindQueryResponseItemsRoot(jresult, ListName);
                }
                catch (Exception ex)
                {
                    OnEnumItemsFailed(ex);
                    throw;
                }
                if (listNode != null)
                {
                    using (ExecutionContextStash.Capture())
                    {
                        foreach (var n in listNode)
                        {
                            yield return(ItemFromJson(n));
                        }
                    }
                }
                // Check for continuation.
                switch (RequestHelper.ParseContinuationParameters(jresult, queryParams, continuationParams))
                {
                case RequestHelper.CONTINUATION_DONE:
                    yield break;

                case RequestHelper.CONTINUATION_AVAILABLE:
                    if (listNode == null)
                    {
                        Site.Logger.LogWarning("Empty query page with continuation received.");
                    }
                    break;

                case RequestHelper.CONTINUATION_LOOP:
                    Site.Logger.LogWarning("Continuation information provided by server response leads to infinite loop. {RawData}",
                                           RequestHelper.FindQueryContinuationParameterRoot(jresult));
                    // The following is just last effort.
                    var outOfLoop = false;
                    if (CompatibilityOptions != null)
                    {
                        if ((CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.FetchMore) ==
                            WikiListContinuationLoopBehaviors.FetchMore)
                        {
                            // xxlimit (length = 7)
                            var limitParamName =
                                queryParams.Keys.FirstOrDefault(k => k.Length == 7 && k.EndsWith("limit", StringComparison.Ordinal));
                            if (limitParamName == null)
                            {
                                Site.Logger.LogWarning("Failed to find the underlying parameter name for PaginationSize.");
                            }
                            else
                            {
                                var maxLimit     = Site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 1000 : 500;
                                var currentLimit = Math.Max(PaginationSize, 50);
                                // Continuously expand PaginationSize, hopefully we can retrieve some different continuation param value.
                                while (currentLimit < maxLimit)
                                {
                                    currentLimit = Math.Min(maxLimit, currentLimit * 2);
                                    Site.Logger.LogDebug("Try to fetch more with {ParamName}={ParamValue}.", limitParamName, currentLimit);
                                    queryParams.Clear();
                                    queryParams.MergeFrom(baseQueryParams);
                                    queryParams.MergeFrom(continuationParams);
                                    queryParams[limitParamName] = currentLimit;
                                    var jresult2 = await Site.InvokeMediaWikiApiAsync(new MediaWikiFormRequestMessage(queryParams),
                                                                                      cancellationToken);

                                    var applyResult = RequestHelper.ParseContinuationParameters(jresult2, queryParams, continuationParams);
                                    switch (applyResult)
                                    {
                                    case RequestHelper.CONTINUATION_AVAILABLE:
                                    case RequestHelper.CONTINUATION_DONE:
                                        var listNode2 = RequestHelper.FindQueryResponseItemsRoot(jresult2, ListName);
                                        Site.Logger.LogInformation("Successfully got out of the continuation loop.");
                                        if (listNode2 != null)
                                        {
                                            if (listNode != null)
                                            {
                                                // Eliminate items that we have already yielded.
                                                var yieldedItems = new HashSet <JToken>(listNode, new JTokenEqualityComparer());
                                                using (ExecutionContextStash.Capture())
                                                    foreach (var n in listNode2.Where(n => !yieldedItems.Contains(n)))
                                                    {
                                                        yield return(ItemFromJson(n));
                                                    }
                                            }
                                            else
                                            {
                                                using (ExecutionContextStash.Capture())
                                                    foreach (var n in listNode2)
                                                    {
                                                        yield return(ItemFromJson(n));
                                                    }
                                            }
                                        }
                                        outOfLoop = true;
                                        if (applyResult == RequestHelper.CONTINUATION_DONE)
                                        {
                                            yield break;
                                        }
                                        break;

                                    case RequestHelper.CONTINUATION_LOOP:
                                        break;
                                    }
                                }
                            }
                        }
                        //if (!outOfLoop && (CompatibilityOptions.ContinuationLoopBehaviors & WikiListContinuationLoopBehaviors.SkipItems) ==
                        //    WikiListContinuationLoopBehaviors.SkipItems)
                        //{

                        //}
                    }
                    if (!outOfLoop)
                    {
                        throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop);
                    }
                    break;
                }
            }
        }