/// <summary>
/// Refresh a sequence of revisions by revid, along with their owner pages.
/// </summary>
/// <remarks>
/// <para>If there's invalid revision id in <paramref name="revIds"/>, an <see cref="ArgumentException"/> will be thrown while enumerating.</para>
/// </remarks>
public static IAsyncEnumerable <Revision> FetchRevisionsAsync(WikiSite site, IEnumerable <int> revIds, PageQueryOptions options, CancellationToken cancellationToken)
{
if (revIds == null)
{
throw new ArgumentNullException(nameof(revIds));
}
var queryParams = GetPageFetchingParams(options);
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
? 500
: 50;
// PageId --> JsonSerializer that can new Revision(Page)
var pagePool = new Dictionary <int, JsonSerializer>();
var revPartition = revIds.Partition(titleLimit).Select(partition => partition.ToList())
.SelectAsync(async partition =>
{
site.Logger?.Trace(site, $"Fetching {partition.Count} revisions.");
queryParams["revids"] = string.Join("|", partition);
var jobj = await site.PostValuesAsync(queryParams, cancellationToken);
var jpages = (JObject)jobj["query"]["pages"];
// Generate converters first
// Use DelegateCreationConverter to create Revision with constructor
var pages = WikiPage.FromJsonQueryResult(site, (JObject)jobj["query"], options);
foreach (var p in pages)
{
if (!pagePool.ContainsKey(p.Id))
{
var p1 = p;
var serializer = Utility.CreateWikiJsonSerializer();
serializer.Converters.Add(new DelegateCreationConverter <Revision>(t => new Revision(p1)));
pagePool.Add(p.Id, serializer);
}
}
// Then convert revisions
var rawRev = jpages.Properties()
.SelectMany(p => p.Value["revisions"].Select(r => new
{
Serializer = pagePool[Convert.ToInt32(p.Name)],
RevisionId = (int)r["revid"],
Revision = r
})).ToDictionary(o => o.RevisionId);
return(partition.Select(revId =>
{
try
{
var raw = rawRev[revId];
return raw.Revision.ToObject <Revision>(raw.Serializer);
}
catch (KeyNotFoundException)
{
throw new ArgumentException($"The revision id {revId} could not be found on the site.",
nameof(revIds));
}
}).ToAsyncEnumerable());
});
return(revPartition.SelectMany(p => p));
}
/// <summary>
/// Asynchronously generates the sequence of pages.
/// </summary>
/// <param name="options">Options when querying for the pages.</param>
public virtual IAsyncEnumerable <WikiPage> EnumPagesAsync(IWikiPageQueryProvider options)
{
var queryParams = options.EnumParameters(Site.SiteInfo.Version).ToDictionary();
queryParams.Add("generator", GeneratorName);
foreach (var v in EnumGeneratorParameters())
{
queryParams[v.Key] = v.Value;
}
return(RequestHelper.QueryWithContinuation(Site, queryParams,
() => Site.BeginActionScope(this, options),
DistinctGeneratedPages)
.SelectMany(jquery => WikiPage.FromJsonQueryResult(Site, jquery, options).ToAsyncEnumerable()));
}
/// <summary>
/// Enumerate pages from the generator.
/// </summary>
public static IAsyncEnumerable <WikiPage> EnumPagesAsync(PageGeneratorBase generator, PageQueryOptions options, int actualPagingSize)
{
if (generator == null)
{
throw new ArgumentNullException(nameof(generator));
}
if ((options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects)
{
throw new ArgumentException("Cannot resolve redirects when using generators.", nameof(options));
}
var queryParams = GetPageFetchingParams(options);
return(generator.EnumJsonAsync(queryParams, actualPagingSize).SelectMany(jresult =>
{
var pages = WikiPage.FromJsonQueryResult(generator.Site, jresult, options);
generator.Site.Logger?.Trace(generator.Site, $"Loaded {pages.Count} pages from {generator}.");
return pages.ToAsyncEnumerable();
}));
}
