/// <summary>
/// Initializes a <see cref="WikiPageQueryProvider"/> from the given <see cref="PageQueryOptions"/> value.
/// </summary>
/// <param name="options">The page query options.</param>
/// <returns>The equivalent <see cref="WikiPageQueryProvider"/> that can be further modified by the caller.</returns>
/// <remarks>If you won't perform any customizations on the returned instance,
/// consider using <see cref="MediaWikiHelper.QueryProviderFromOptions"/>.</remarks>
public static WikiPageQueryProvider FromOptions(PageQueryOptions options)
{
if ((options & (PageQueryOptions.FetchContent | PageQueryOptions.ResolveRedirects)) != options)
{
throw new ArgumentException(Prompts.ExceptionInvalidEnumValue, nameof(options));
}
var provider = new WikiPageQueryProvider
{
Properties = new List <IWikiPagePropertyProvider <IWikiPagePropertyGroup> >
{
new PageInfoPropertyProvider {
},
new RevisionsPropertyProvider {
FetchContent = (options & PageQueryOptions.FetchContent) == PageQueryOptions.FetchContent
},
new CategoryInfoPropertyProvider {
},
new PagePropertiesPropertyProvider {
},
new FileInfoPropertyProvider {
},
},
ResolveRedirects = (options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects
};
return(provider);
}
public IPagedList <FinancialConcertMarketingAgreeAdminViewModel> Get([FromUri] PageQueryOptions options)
{
var list = service.GetAll();
var data = mapperConfig.CreateMapper().Map <IList <FinancialConcertMarketingAgreeAdminViewModel> >(list);
return(new SerializablePagedList <FinancialConcertMarketingAgreeAdminViewModel>(data, options.Page, options.PageSize));
}
public IPagedList <FinancialConsultantSharingNewViewModel> GetNew([FromUri] PageQueryOptions options)
{
var list = financialConsultantSharingService.GetAdminFinancialConsultantNewEntryList();
var data = mapperConfig.CreateMapper().Map <IList <FinancialConsultantSharingNewViewModel> >(list);
return(new SerializablePagedList <FinancialConsultantSharingNewViewModel>(data, options.Page, options.PageSize));
}
/// <summary>
/// Asynchornously generate the sequence of revisions.
/// </summary>
/// <param name="options">Options when querying for the revisions. Note <see cref="PageQueryOptions.ResolveRedirects"/> will raise exception.</param>
/// <exception cref="ArgumentException"><see cref="PageQueryOptions.ResolveRedirects"/> is set.</exception>
/// <exception cref="InvalidOperationException"><see cref="Page"/> is <c>null</c>.</exception>
public IAsyncEnumerable <Revision> EnumRevisionsAsync(PageQueryOptions options)
{
// We do not resolve redirects.
if ((options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects)
{
throw new ArgumentException("Cannot resolve redirects when querying for revisions.", nameof(options));
}
return(RequestHelper.EnumRevisionsAsync(this, options));
}
/// <summary>
/// Gets the actual value of <see cref="PagingSize"/> used for request.
/// </summary>
/// <param name="options">The options used when attempting to enumerate the pages.</param>
/// <returns>
/// The estimated items (i.e. wiki pages) count per page.
/// If <see cref="PagingSize"/> is set, then the returned value will be identical to it.
/// </returns>
/// <remarks>
/// If <see cref="PagingSize"/> is <c>null</c>, and <see cref="PageQueryOptions.FetchContent"/> is specified,
/// the default limit will be 1/10 of the original default limit (500 for bots and 50 for users).
/// (See https://www.mediawiki.org/wiki/API:Revisions .)
/// If you have manually set <see cref="PagingSize"/>, this function will directly return the value you have set,
/// but any value exceeding the server limit will case problems, such as empty content retrieved (even if
/// you have set <see cref="PageQueryOptions.FetchContent"/>), or <see cref="WikiClientException"/>.
/// </remarks>
public int GetActualPagingSize(PageQueryOptions options)
{
if (PagingSize != null)
{
return(PagingSize.Value);
}
return((options & PageQueryOptions.FetchContent) == PageQueryOptions.FetchContent
? Site.ListingPagingSize / 10
: Site.ListingPagingSize);
}
/// <summary>
/// Enumerates revisions of the page, descending in tim.
/// </summary>
/// <param name="pagingSize">Maximum items returned per request. <c>null</c> for maximum allowed count.</param>
/// <param name="options">Options for revision listing. Note <see cref="PageQueryOptions.ResolveRedirects"/> will raise exception.</param>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="pagingSize"/> is non-positive.</exception>
/// <remarks>To gain full control of revision enumeration, you can use <see cref="RevisionGenerator" />.</remarks>
public IAsyncEnumerable <Revision> EnumRevisionsAsync(int?pagingSize, PageQueryOptions options)
{
if (pagingSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(pagingSize));
}
var gen = new RevisionGenerator(this)
{
PagingSize = pagingSize
};
return(gen.EnumRevisionsAsync(options));
}
/// <summary>
/// Loads page information from JSON.
/// </summary>
/// <param name="prop">query.pages.xxx property.</param>
/// <param name="options">Provides options when performing the query.</param>
internal void LoadFromJson(JProperty prop, PageQueryOptions options)
{
var id = Convert.ToInt32(prop.Name);
// I'm not sure whether this assertion holds.
Debug.Assert(id != 0);
if ((options & PageQueryOptions.ResolveRedirects) != PageQueryOptions.ResolveRedirects &&
Id != 0 && !AreIdEquals(Id, id))
{
// The page has been overwritten, or deleted.
WikiClient.Logger?.Warn(this, $"Detected change of page id: {Title}, {Id}");
}
Id = id;
var page = (JObject)prop.Value;
OnLoadPageInfo(page);
// TODO Cache content
LoadLastRevision(page);
QueryOptions = options;
}
/// <summary>
/// Builds common parameters for fetching a page.
/// </summary>
private static IDictionary <string, object> GetPageFetchingParams(PageQueryOptions options)
{
var queryParams = new Dictionary <string, object>
{
{ "action", "query" },
// We also fetch category info, just in case.
{ "prop", "info|categoryinfo|imageinfo|revisions|pageprops" },
{ "inprop", "protection" },
{ "iiprop", "timestamp|user|comment|url|size|sha1" },
{ "rvprop", "ids|timestamp|flags|comment|user|contentmodel|sha1|tags|size" },
{ "redirects", (options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects },
{ "maxlag", 5 },
};
if ((options & PageQueryOptions.FetchContent) == PageQueryOptions.FetchContent)
{
queryParams["rvprop"] += "|content";
}
return(queryParams);
}
/// <summary>
/// Creates a list of <see cref="WikiPage"/> based on JSON query result.
/// </summary>
/// <param name="site">A <see cref="Site"/> object.</param>
/// <param name="queryNode">The <c>qurey</c> node value object of JSON result.</param>
/// <param name="options">Provides options when performing the query.</param>
/// <returns>Retrived pages.</returns>
internal static IList <WikiPage> FromJsonQueryResult(WikiSite site, JObject queryNode, PageQueryOptions options)
{
if (site == null)
{
throw new ArgumentNullException(nameof(site));
}
if (queryNode == null)
{
throw new ArgumentNullException(nameof(queryNode));
}
var pages = (JObject)queryNode["pages"];
if (pages == null)
{
return(EmptyPages);
}
// If query.xxx.index exists, sort the pages by the given index.
// This is specifically used with SearchGenerator, to keep the search result in order.
// For other generators, this property simply does not exist.
// See https://www.mediawiki.org/wiki/API_talk:Query#On_the_order_of_titles_taken_out_of_generator .
return(pages.Properties().OrderBy(page => (int?)page.Value["index"])
.Select(page =>
{
WikiPage newInst;
if (page.Value["categoryinfo"] != null)
{
newInst = new CategoryPage(site);
}
else if ((string)page.Value["contentmodel"] == ContentModels.FlowBoard)
{
if ((int)page.Value["ns"] == FlowNamespaces.Topic)
{
newInst = new Topic(site);
}
else
{
newInst = new Board(site);
}
}
else
{
newInst = new WikiPage(site);
}
newInst.LoadFromJson(page, options);
return newInst;
}).ToList());
}
/// <summary>
/// Gets a read-only implementation of <see cref="IWikiPageQueryProvider"/> for fetching a page.
/// </summary>
/// <remarks>
/// This method returns a shared read-only instance for a specific <see cref="PageQueryOptions"/> value to reduce memory consumption.
/// If you want to apply your customization based on the presets, use <see cref="WikiPageQueryProvider.FromOptions"/>.
/// </remarks>
public static IWikiPageQueryProvider QueryProviderFromOptions(PageQueryOptions options)
{
return(queryProviderPresets.GetOrAdd(options,
k => new SealedWikiPageQueryProvider(WikiPageQueryProvider.FromOptions(options))));
}
/// <summary>
/// Asynchronously generates the sequence of pages.
/// </summary>
public IAsyncEnumerable <WikiPage> EnumPagesAsync(PageQueryOptions options)
{
return(EnumPagesAsync(MediaWikiHelper.QueryProviderFromOptions(options)));
}
public IEnumerable <WikiPage> EnumMembers(PageQueryOptions options)
{
return(new CategoryMembersGenerator(Site, Title).EnumPages(options));
}
public IPagedList <NilririmanboEntry> GetAdminNilririmamboEntryList([FromUri] PageQueryOptions options)
{
var result = service.GetAdminNilririmamboEntryList();
return(new SerializablePagedList <NilririmanboEntry>(result, options.Page, options.PageSize));
}
/// <summary>
/// Enumerate pages from the generator.
/// </summary>
public static IAsyncEnumerable <WikiPage> EnumPagesAsync(PageGeneratorBase generator, PageQueryOptions options, int actualPagingSize)
{
if (generator == null)
{
throw new ArgumentNullException(nameof(generator));
}
if ((options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects)
{
throw new ArgumentException("Cannot resolve redirects when using generators.", nameof(options));
}
var queryParams = GetPageFetchingParams(options);
return(generator.EnumJsonAsync(queryParams, actualPagingSize).SelectMany(jresult =>
{
var pages = WikiPage.FromJsonQueryResult(generator.Site, jresult, options);
generator.Site.Logger?.Trace(generator.Site, $"Loaded {pages.Count} pages from {generator}.");
return pages.ToAsyncEnumerable();
}));
}
/// <summary>
/// Fetch information for the page.
/// </summary>
/// <param name="options">Options when querying for the pages.</param>
/// <param name="cancellationToken">The cancellation token that will be checked prior to completing the returned task.</param>
/// <remarks>
/// For fetching multiple pages at one time, see <see cref="PageExtensions.RefreshAsync(IEnumerable{WikiPage}, PageQueryOptions)"/>.
/// </remarks>
/// <exception cref="InvalidOperationException">Circular redirect detected when resolving redirects.</exception>
public Task RefreshAsync(PageQueryOptions options, CancellationToken cancellationToken)
{
return(RequestHelper.RefreshPagesAsync(new[] { this }, options, cancellationToken));
}
/// <summary>
/// Asynchronously fetch information for a sequence of pages.
/// </summary>
/// <param name="pages">A sequence of pages to be refreshed.</param>
/// <param name="options">Provides options when performing the query.</param>
/// <remarks>
/// It's recommended that <paramref name="pages"/> is a list or a subset of a list
/// that is hold by caller, because this method will not return the refreshed pages.
/// </remarks>
/// <exception cref="InvalidOperationException">Circular redirect detected when resolving redirects.</exception>
public static Task RefreshAsync(this IEnumerable <WikiPage> pages, PageQueryOptions options)
{
return(RefreshAsync(pages, options, new CancellationToken()));
}
/// <summary>
/// Refresh a sequence of pages.
/// </summary>
public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, PageQueryOptions options, CancellationToken cancellationToken)
{
if (pages == null)
{
throw new ArgumentNullException(nameof(pages));
}
// You can even fetch pages from different sites.
foreach (var sitePages in pages.GroupBy(p => Tuple.Create(p.Site, p.GetType())))
{
var site = sitePages.Key.Item1;
var queryParams = GetPageFetchingParams(options);
var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits)
? 500
: 50;
foreach (var partition in sitePages.Partition(titleLimit).Select(partition => partition.ToList()))
{
site.Logger?.Trace(site, $"Fetching {partition.Count} pages.");
// We use titles to query pages.
queryParams["titles"] = string.Join("|", partition.Select(p => p.Title));
var jobj = await site.PostValuesAsync(queryParams, cancellationToken);
// Process title normalization.
var normalized = jobj["query"]["normalized"]?.ToDictionary(n => (string)n["from"],
n => (string)n["to"]);
// Process redirects.
var redirects = jobj["query"]["redirects"]?.ToDictionary(n => (string)n["from"],
n => (string)n["to"]);
var pageInfoDict = ((JObject)jobj["query"]["pages"]).Properties()
.ToDictionary(p => p.Value["title"]);
foreach (var page in partition)
{
var title = page.Title;
// Normalize the title first.
if (normalized?.ContainsKey(title) ?? false)
{
title = normalized[title];
}
// Then process the redirects.
var redirectTrace = new List <string>();
while (redirects?.ContainsKey(title) ?? false)
{
redirectTrace.Add(title); // Adds the last title
var next = redirects[title];
if (redirectTrace.Contains(next))
{
throw new InvalidOperationException(
$"Cannot resolve circular redirect: {string.Join("->", redirectTrace)}.");
}
title = next;
}
// Finally, get the page.
var pageInfo = pageInfoDict[title];
if (redirectTrace.Count > 0)
{
page.RedirectPath = redirectTrace;
}
page.LoadFromJson(pageInfo, options);
}
}
}
}
public override IAsyncEnumerable <WikiPage> EnumPagesAsync(PageQueryOptions options)
{
return(base.EnumPagesAsync(options));
}
/// <inheritdoc cref="RefreshAsync(IEnumerable{WikiPage},IWikiPageQueryProvider,CancellationToken)"/>
public static Task RefreshAsync(this IEnumerable <WikiPage> pages, PageQueryOptions options, CancellationToken cancellationToken)
{
return(RequestHelper.RefreshPagesAsync(pages, MediaWikiHelper.QueryProviderFromOptions(options), cancellationToken));
}
/// <inheritdoc cref="RefreshAsync(IWikiPageQueryProvider, CancellationToken)"/>
public Task RefreshAsync(PageQueryOptions options, CancellationToken cancellationToken)
{
return(RefreshAsync(MediaWikiHelper.QueryProviderFromOptions(options), cancellationToken));
}
/// <inheritdoc cref="RefreshAsync(IWikiPageQueryProvider, CancellationToken)"/>
public Task RefreshAsync(PageQueryOptions options)
{
return(RefreshAsync(options, CancellationToken.None));
}
public PagedList.IPagedList <AdminMarvelFrozenStatsViewModel> GetMarvelFrozenSNSStats([FromUri] PageQueryOptions options)
{
var list = service.GetStatistics().OrderByDescending(x => x.TotalCount);
return(new SerializablePagedList <AdminMarvelFrozenStatsViewModel>(list, options.Page, options.PageSize));
}
/// <summary>
/// Asynchronously fetch information for a sequence of pages.
/// </summary>
/// <param name="pages">A sequence of pages to be refreshed.</param>
/// <param name="options">Provides options when performing the query.</param>
/// <param name="cancellationToken">The cancellation token that will be checked prior to completing the returned task.</param>
/// <remarks>
/// It's recommended that <paramref name="pages"/> is a list or a subset of a list
/// that is hold by caller, because this method will not return the refreshed pages.
/// </remarks>
/// <exception cref="InvalidOperationException">Circular redirect detected when resolving redirects.</exception>
public static Task RefreshAsync(this IEnumerable <WikiPage> pages, PageQueryOptions options, CancellationToken cancellationToken)
{
return(RequestHelper.RefreshPagesAsync(pages, options, cancellationToken));
}
/// <inheritdoc cref="FetchRevisionsAsync(WikiSite,IEnumerable{int},IWikiPageQueryProvider,CancellationToken)"/>
public static IAsyncEnumerable <Revision> FetchRevisionsAsync(WikiSite site, IEnumerable <int> revisionIds, PageQueryOptions options)
{
return(FetchRevisionsAsync(site, revisionIds, options, CancellationToken.None));
}
public IPagedList <AdminMainStreamSnsStatsViewModel> GetMainStreamSnsStatsModel([FromUri] PageQueryOptions options)
{
var list = service.GetMainStreamSurpriseSnsStats().OrderByDescending(x => x.TotalCount).ToList();
return(new SerializablePagedList <AdminMainStreamSnsStatsViewModel>(list, options.Page, options.PageSize));
}
/// <summary>
/// Asynchornously generate the sequence of pages.
/// </summary>
/// <param name="options">Options when querying for the pages.</param>
public IAsyncEnumerable <T> EnumPagesAsync(PageQueryOptions options)
{
var actualPagingSize = GetActualPagingSize(options);
return(RequestHelper.EnumPagesAsync(this, options, actualPagingSize).Cast <T>());
}
public IPagedList <KidsNoteEntry> GetAdminKidsNoteEntryList([FromUri] PageQueryOptions options)
{
var result = service.GetAdminKidsNoteEntryList();
return(new SerializablePagedList <KidsNoteEntry>(result, options.Page, options.PageSize));
}
/// <summary>
/// Fetch information for the page.
/// </summary>
/// <param name="options">Options when querying for the pages.</param>
/// <remarks>
/// For fetching multiple pages at one time, see <see cref="PageExtensions.RefreshAsync(IEnumerable{WikiPage}, PageQueryOptions)"/>.
/// </remarks>
/// <exception cref="InvalidOperationException">Circular redirect detected when resolving redirects.</exception>
public Task RefreshAsync(PageQueryOptions options)
{
return(RefreshAsync(options, new CancellationToken()));
}
/// <inheritdoc cref="FetchRevisionsAsync(WikiSite,IEnumerable{int},IWikiPageQueryProvider,CancellationToken)"/>
public static IAsyncEnumerable <Revision> FetchRevisionsAsync(WikiSite site, IEnumerable <int> revisionIds, PageQueryOptions options, CancellationToken cancellationToken)
{
return(FetchRevisionsAsync(site, revisionIds, MediaWikiHelper.QueryProviderFromOptions(options), cancellationToken));
}
/// <summary>
/// Synchornously generate the sequence of pages.
/// </summary>
/// <param name="options">Options when querying for the pages.</param>
public IEnumerable <T> EnumPages(PageQueryOptions options)
{
return(EnumPagesAsync(options).ToEnumerable());
}
