/// <summary> /// Refresh a sequence of revisions by revid, along with their owner pages. /// </summary> /// <remarks> /// <para>If there's invalid revision id in <paramref name="revIds"/>, an <see cref="ArgumentException"/> will be thrown while enumerating.</para> /// </remarks> public static IAsyncEnumerable <Revision> FetchRevisionsAsync(WikiSite site, IEnumerable <int> revIds, PageQueryOptions options, CancellationToken cancellationToken) { if (revIds == null) { throw new ArgumentNullException(nameof(revIds)); } var queryParams = GetPageFetchingParams(options); var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; // PageId --> JsonSerializer that can new Revision(Page) var pagePool = new Dictionary <int, JsonSerializer>(); var revPartition = revIds.Partition(titleLimit).Select(partition => partition.ToList()) .SelectAsync(async partition => { site.Logger?.Trace(site, $"Fetching {partition.Count} revisions."); queryParams["revids"] = string.Join("|", partition); var jobj = await site.PostValuesAsync(queryParams, cancellationToken); var jpages = (JObject)jobj["query"]["pages"]; // Generate converters first // Use DelegateCreationConverter to create Revision with constructor var pages = WikiPage.FromJsonQueryResult(site, (JObject)jobj["query"], options); foreach (var p in pages) { if (!pagePool.ContainsKey(p.Id)) { var p1 = p; var serializer = Utility.CreateWikiJsonSerializer(); serializer.Converters.Add(new DelegateCreationConverter <Revision>(t => new Revision(p1))); pagePool.Add(p.Id, serializer); } } // Then convert revisions var rawRev = jpages.Properties() .SelectMany(p => p.Value["revisions"].Select(r => new { Serializer = pagePool[Convert.ToInt32(p.Name)], RevisionId = (int)r["revid"], Revision = r })).ToDictionary(o => o.RevisionId); return(partition.Select(revId => { try { var raw = rawRev[revId]; return raw.Revision.ToObject <Revision>(raw.Serializer); } catch (KeyNotFoundException) { throw new ArgumentException($"The revision id {revId} could not be found on the site.", nameof(revIds)); } }).ToAsyncEnumerable()); }); return(revPartition.SelectMany(p => p)); }
/// <summary> /// Asynchronously generates the sequence of pages. /// </summary> /// <param name="options">Options when querying for the pages.</param> public virtual IAsyncEnumerable <WikiPage> EnumPagesAsync(IWikiPageQueryProvider options) { var queryParams = options.EnumParameters(Site.SiteInfo.Version).ToDictionary(); queryParams.Add("generator", GeneratorName); foreach (var v in EnumGeneratorParameters()) { queryParams[v.Key] = v.Value; } return(RequestHelper.QueryWithContinuation(Site, queryParams, () => Site.BeginActionScope(this, options), DistinctGeneratedPages) .SelectMany(jquery => WikiPage.FromJsonQueryResult(Site, jquery, options).ToAsyncEnumerable())); }
/// <summary> /// Enumerate pages from the generator. /// </summary> public static IAsyncEnumerable <WikiPage> EnumPagesAsync(PageGeneratorBase generator, PageQueryOptions options, int actualPagingSize) { if (generator == null) { throw new ArgumentNullException(nameof(generator)); } if ((options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects) { throw new ArgumentException("Cannot resolve redirects when using generators.", nameof(options)); } var queryParams = GetPageFetchingParams(options); return(generator.EnumJsonAsync(queryParams, actualPagingSize).SelectMany(jresult => { var pages = WikiPage.FromJsonQueryResult(generator.Site, jresult, options); generator.Site.Logger?.Trace(generator.Site, $"Loaded {pages.Count} pages from {generator}."); return pages.ToAsyncEnumerable(); })); }