/// <summary> /// Initializes a <see cref="WikiPageQueryProvider"/> from the given <see cref="PageQueryOptions"/> value. /// </summary> /// <param name="options">The page query options.</param> /// <returns>The equivalent <see cref="WikiPageQueryProvider"/> that can be further modified by the caller.</returns> /// <remarks>If you won't perform any customizations on the returned instance, /// consider using <see cref="MediaWikiHelper.QueryProviderFromOptions"/>.</remarks> public static WikiPageQueryProvider FromOptions(PageQueryOptions options) { if ((options & (PageQueryOptions.FetchContent | PageQueryOptions.ResolveRedirects)) != options) { throw new ArgumentException(Prompts.ExceptionInvalidEnumValue, nameof(options)); } var provider = new WikiPageQueryProvider { Properties = new List <IWikiPagePropertyProvider <IWikiPagePropertyGroup> > { new PageInfoPropertyProvider { }, new RevisionsPropertyProvider { FetchContent = (options & PageQueryOptions.FetchContent) == PageQueryOptions.FetchContent }, new CategoryInfoPropertyProvider { }, new PagePropertiesPropertyProvider { }, new FileInfoPropertyProvider { }, }, ResolveRedirects = (options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects }; return(provider); }
public IPagedList <FinancialConcertMarketingAgreeAdminViewModel> Get([FromUri] PageQueryOptions options) { var list = service.GetAll(); var data = mapperConfig.CreateMapper().Map <IList <FinancialConcertMarketingAgreeAdminViewModel> >(list); return(new SerializablePagedList <FinancialConcertMarketingAgreeAdminViewModel>(data, options.Page, options.PageSize)); }
public IPagedList <FinancialConsultantSharingNewViewModel> GetNew([FromUri] PageQueryOptions options) { var list = financialConsultantSharingService.GetAdminFinancialConsultantNewEntryList(); var data = mapperConfig.CreateMapper().Map <IList <FinancialConsultantSharingNewViewModel> >(list); return(new SerializablePagedList <FinancialConsultantSharingNewViewModel>(data, options.Page, options.PageSize)); }
/// <summary> /// Asynchornously generate the sequence of revisions. /// </summary> /// <param name="options">Options when querying for the revisions. Note <see cref="PageQueryOptions.ResolveRedirects"/> will raise exception.</param> /// <exception cref="ArgumentException"><see cref="PageQueryOptions.ResolveRedirects"/> is set.</exception> /// <exception cref="InvalidOperationException"><see cref="Page"/> is <c>null</c>.</exception> public IAsyncEnumerable <Revision> EnumRevisionsAsync(PageQueryOptions options) { // We do not resolve redirects. if ((options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects) { throw new ArgumentException("Cannot resolve redirects when querying for revisions.", nameof(options)); } return(RequestHelper.EnumRevisionsAsync(this, options)); }
/// <summary> /// Gets the actual value of <see cref="PagingSize"/> used for request. /// </summary> /// <param name="options">The options used when attempting to enumerate the pages.</param> /// <returns> /// The estimated items (i.e. wiki pages) count per page. /// If <see cref="PagingSize"/> is set, then the returned value will be identical to it. /// </returns> /// <remarks> /// If <see cref="PagingSize"/> is <c>null</c>, and <see cref="PageQueryOptions.FetchContent"/> is specified, /// the default limit will be 1/10 of the original default limit (500 for bots and 50 for users). /// (See https://www.mediawiki.org/wiki/API:Revisions .) /// If you have manually set <see cref="PagingSize"/>, this function will directly return the value you have set, /// but any value exceeding the server limit will case problems, such as empty content retrieved (even if /// you have set <see cref="PageQueryOptions.FetchContent"/>), or <see cref="WikiClientException"/>. /// </remarks> public int GetActualPagingSize(PageQueryOptions options) { if (PagingSize != null) { return(PagingSize.Value); } return((options & PageQueryOptions.FetchContent) == PageQueryOptions.FetchContent ? Site.ListingPagingSize / 10 : Site.ListingPagingSize); }
/// <summary> /// Enumerates revisions of the page, descending in tim. /// </summary> /// <param name="pagingSize">Maximum items returned per request. <c>null</c> for maximum allowed count.</param> /// <param name="options">Options for revision listing. Note <see cref="PageQueryOptions.ResolveRedirects"/> will raise exception.</param> /// <exception cref="ArgumentOutOfRangeException"><paramref name="pagingSize"/> is non-positive.</exception> /// <remarks>To gain full control of revision enumeration, you can use <see cref="RevisionGenerator" />.</remarks> public IAsyncEnumerable <Revision> EnumRevisionsAsync(int?pagingSize, PageQueryOptions options) { if (pagingSize <= 0) { throw new ArgumentOutOfRangeException(nameof(pagingSize)); } var gen = new RevisionGenerator(this) { PagingSize = pagingSize }; return(gen.EnumRevisionsAsync(options)); }
/// <summary> /// Loads page information from JSON. /// </summary> /// <param name="prop">query.pages.xxx property.</param> /// <param name="options">Provides options when performing the query.</param> internal void LoadFromJson(JProperty prop, PageQueryOptions options) { var id = Convert.ToInt32(prop.Name); // I'm not sure whether this assertion holds. Debug.Assert(id != 0); if ((options & PageQueryOptions.ResolveRedirects) != PageQueryOptions.ResolveRedirects && Id != 0 && !AreIdEquals(Id, id)) { // The page has been overwritten, or deleted. WikiClient.Logger?.Warn(this, $"Detected change of page id: {Title}, {Id}"); } Id = id; var page = (JObject)prop.Value; OnLoadPageInfo(page); // TODO Cache content LoadLastRevision(page); QueryOptions = options; }
/// <summary> /// Builds common parameters for fetching a page. /// </summary> private static IDictionary <string, object> GetPageFetchingParams(PageQueryOptions options) { var queryParams = new Dictionary <string, object> { { "action", "query" }, // We also fetch category info, just in case. { "prop", "info|categoryinfo|imageinfo|revisions|pageprops" }, { "inprop", "protection" }, { "iiprop", "timestamp|user|comment|url|size|sha1" }, { "rvprop", "ids|timestamp|flags|comment|user|contentmodel|sha1|tags|size" }, { "redirects", (options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects }, { "maxlag", 5 }, }; if ((options & PageQueryOptions.FetchContent) == PageQueryOptions.FetchContent) { queryParams["rvprop"] += "|content"; } return(queryParams); }
/// <summary> /// Creates a list of <see cref="WikiPage"/> based on JSON query result. /// </summary> /// <param name="site">A <see cref="Site"/> object.</param> /// <param name="queryNode">The <c>qurey</c> node value object of JSON result.</param> /// <param name="options">Provides options when performing the query.</param> /// <returns>Retrived pages.</returns> internal static IList <WikiPage> FromJsonQueryResult(WikiSite site, JObject queryNode, PageQueryOptions options) { if (site == null) { throw new ArgumentNullException(nameof(site)); } if (queryNode == null) { throw new ArgumentNullException(nameof(queryNode)); } var pages = (JObject)queryNode["pages"]; if (pages == null) { return(EmptyPages); } // If query.xxx.index exists, sort the pages by the given index. // This is specifically used with SearchGenerator, to keep the search result in order. // For other generators, this property simply does not exist. // See https://www.mediawiki.org/wiki/API_talk:Query#On_the_order_of_titles_taken_out_of_generator . return(pages.Properties().OrderBy(page => (int?)page.Value["index"]) .Select(page => { WikiPage newInst; if (page.Value["categoryinfo"] != null) { newInst = new CategoryPage(site); } else if ((string)page.Value["contentmodel"] == ContentModels.FlowBoard) { if ((int)page.Value["ns"] == FlowNamespaces.Topic) { newInst = new Topic(site); } else { newInst = new Board(site); } } else { newInst = new WikiPage(site); } newInst.LoadFromJson(page, options); return newInst; }).ToList()); }
/// <summary> /// Gets a read-only implementation of <see cref="IWikiPageQueryProvider"/> for fetching a page. /// </summary> /// <remarks> /// This method returns a shared read-only instance for a specific <see cref="PageQueryOptions"/> value to reduce memory consumption. /// If you want to apply your customization based on the presets, use <see cref="WikiPageQueryProvider.FromOptions"/>. /// </remarks> public static IWikiPageQueryProvider QueryProviderFromOptions(PageQueryOptions options) { return(queryProviderPresets.GetOrAdd(options, k => new SealedWikiPageQueryProvider(WikiPageQueryProvider.FromOptions(options)))); }
/// <summary> /// Asynchronously generates the sequence of pages. /// </summary> public IAsyncEnumerable <WikiPage> EnumPagesAsync(PageQueryOptions options) { return(EnumPagesAsync(MediaWikiHelper.QueryProviderFromOptions(options))); }
public IEnumerable <WikiPage> EnumMembers(PageQueryOptions options) { return(new CategoryMembersGenerator(Site, Title).EnumPages(options)); }
public IPagedList <NilririmanboEntry> GetAdminNilririmamboEntryList([FromUri] PageQueryOptions options) { var result = service.GetAdminNilririmamboEntryList(); return(new SerializablePagedList <NilririmanboEntry>(result, options.Page, options.PageSize)); }
/// <summary> /// Enumerate pages from the generator. /// </summary> public static IAsyncEnumerable <WikiPage> EnumPagesAsync(PageGeneratorBase generator, PageQueryOptions options, int actualPagingSize) { if (generator == null) { throw new ArgumentNullException(nameof(generator)); } if ((options & PageQueryOptions.ResolveRedirects) == PageQueryOptions.ResolveRedirects) { throw new ArgumentException("Cannot resolve redirects when using generators.", nameof(options)); } var queryParams = GetPageFetchingParams(options); return(generator.EnumJsonAsync(queryParams, actualPagingSize).SelectMany(jresult => { var pages = WikiPage.FromJsonQueryResult(generator.Site, jresult, options); generator.Site.Logger?.Trace(generator.Site, $"Loaded {pages.Count} pages from {generator}."); return pages.ToAsyncEnumerable(); })); }
/// <summary> /// Fetch information for the page. /// </summary> /// <param name="options">Options when querying for the pages.</param> /// <param name="cancellationToken">The cancellation token that will be checked prior to completing the returned task.</param> /// <remarks> /// For fetching multiple pages at one time, see <see cref="PageExtensions.RefreshAsync(IEnumerable{WikiPage}, PageQueryOptions)"/>. /// </remarks> /// <exception cref="InvalidOperationException">Circular redirect detected when resolving redirects.</exception> public Task RefreshAsync(PageQueryOptions options, CancellationToken cancellationToken) { return(RequestHelper.RefreshPagesAsync(new[] { this }, options, cancellationToken)); }
/// <summary> /// Asynchronously fetch information for a sequence of pages. /// </summary> /// <param name="pages">A sequence of pages to be refreshed.</param> /// <param name="options">Provides options when performing the query.</param> /// <remarks> /// It's recommended that <paramref name="pages"/> is a list or a subset of a list /// that is hold by caller, because this method will not return the refreshed pages. /// </remarks> /// <exception cref="InvalidOperationException">Circular redirect detected when resolving redirects.</exception> public static Task RefreshAsync(this IEnumerable <WikiPage> pages, PageQueryOptions options) { return(RefreshAsync(pages, options, new CancellationToken())); }
/// <summary> /// Refresh a sequence of pages. /// </summary> public static async Task RefreshPagesAsync(IEnumerable <WikiPage> pages, PageQueryOptions options, CancellationToken cancellationToken) { if (pages == null) { throw new ArgumentNullException(nameof(pages)); } // You can even fetch pages from different sites. foreach (var sitePages in pages.GroupBy(p => Tuple.Create(p.Site, p.GetType()))) { var site = sitePages.Key.Item1; var queryParams = GetPageFetchingParams(options); var titleLimit = site.AccountInfo.HasRight(UserRights.ApiHighLimits) ? 500 : 50; foreach (var partition in sitePages.Partition(titleLimit).Select(partition => partition.ToList())) { site.Logger?.Trace(site, $"Fetching {partition.Count} pages."); // We use titles to query pages. queryParams["titles"] = string.Join("|", partition.Select(p => p.Title)); var jobj = await site.PostValuesAsync(queryParams, cancellationToken); // Process title normalization. var normalized = jobj["query"]["normalized"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); // Process redirects. var redirects = jobj["query"]["redirects"]?.ToDictionary(n => (string)n["from"], n => (string)n["to"]); var pageInfoDict = ((JObject)jobj["query"]["pages"]).Properties() .ToDictionary(p => p.Value["title"]); foreach (var page in partition) { var title = page.Title; // Normalize the title first. if (normalized?.ContainsKey(title) ?? false) { title = normalized[title]; } // Then process the redirects. var redirectTrace = new List <string>(); while (redirects?.ContainsKey(title) ?? false) { redirectTrace.Add(title); // Adds the last title var next = redirects[title]; if (redirectTrace.Contains(next)) { throw new InvalidOperationException( $"Cannot resolve circular redirect: {string.Join("->", redirectTrace)}."); } title = next; } // Finally, get the page. var pageInfo = pageInfoDict[title]; if (redirectTrace.Count > 0) { page.RedirectPath = redirectTrace; } page.LoadFromJson(pageInfo, options); } } } }
public override IAsyncEnumerable <WikiPage> EnumPagesAsync(PageQueryOptions options) { return(base.EnumPagesAsync(options)); }
/// <inheritdoc cref="RefreshAsync(IEnumerable{WikiPage},IWikiPageQueryProvider,CancellationToken)"/> public static Task RefreshAsync(this IEnumerable <WikiPage> pages, PageQueryOptions options, CancellationToken cancellationToken) { return(RequestHelper.RefreshPagesAsync(pages, MediaWikiHelper.QueryProviderFromOptions(options), cancellationToken)); }
/// <inheritdoc cref="RefreshAsync(IWikiPageQueryProvider, CancellationToken)"/> public Task RefreshAsync(PageQueryOptions options, CancellationToken cancellationToken) { return(RefreshAsync(MediaWikiHelper.QueryProviderFromOptions(options), cancellationToken)); }
/// <inheritdoc cref="RefreshAsync(IWikiPageQueryProvider, CancellationToken)"/> public Task RefreshAsync(PageQueryOptions options) { return(RefreshAsync(options, CancellationToken.None)); }
public PagedList.IPagedList <AdminMarvelFrozenStatsViewModel> GetMarvelFrozenSNSStats([FromUri] PageQueryOptions options) { var list = service.GetStatistics().OrderByDescending(x => x.TotalCount); return(new SerializablePagedList <AdminMarvelFrozenStatsViewModel>(list, options.Page, options.PageSize)); }
/// <summary> /// Asynchronously fetch information for a sequence of pages. /// </summary> /// <param name="pages">A sequence of pages to be refreshed.</param> /// <param name="options">Provides options when performing the query.</param> /// <param name="cancellationToken">The cancellation token that will be checked prior to completing the returned task.</param> /// <remarks> /// It's recommended that <paramref name="pages"/> is a list or a subset of a list /// that is hold by caller, because this method will not return the refreshed pages. /// </remarks> /// <exception cref="InvalidOperationException">Circular redirect detected when resolving redirects.</exception> public static Task RefreshAsync(this IEnumerable <WikiPage> pages, PageQueryOptions options, CancellationToken cancellationToken) { return(RequestHelper.RefreshPagesAsync(pages, options, cancellationToken)); }
/// <inheritdoc cref="FetchRevisionsAsync(WikiSite,IEnumerable{int},IWikiPageQueryProvider,CancellationToken)"/> public static IAsyncEnumerable <Revision> FetchRevisionsAsync(WikiSite site, IEnumerable <int> revisionIds, PageQueryOptions options) { return(FetchRevisionsAsync(site, revisionIds, options, CancellationToken.None)); }
public IPagedList <AdminMainStreamSnsStatsViewModel> GetMainStreamSnsStatsModel([FromUri] PageQueryOptions options) { var list = service.GetMainStreamSurpriseSnsStats().OrderByDescending(x => x.TotalCount).ToList(); return(new SerializablePagedList <AdminMainStreamSnsStatsViewModel>(list, options.Page, options.PageSize)); }
/// <summary> /// Asynchornously generate the sequence of pages. /// </summary> /// <param name="options">Options when querying for the pages.</param> public IAsyncEnumerable <T> EnumPagesAsync(PageQueryOptions options) { var actualPagingSize = GetActualPagingSize(options); return(RequestHelper.EnumPagesAsync(this, options, actualPagingSize).Cast <T>()); }
public IPagedList <KidsNoteEntry> GetAdminKidsNoteEntryList([FromUri] PageQueryOptions options) { var result = service.GetAdminKidsNoteEntryList(); return(new SerializablePagedList <KidsNoteEntry>(result, options.Page, options.PageSize)); }
/// <summary> /// Fetch information for the page. /// </summary> /// <param name="options">Options when querying for the pages.</param> /// <remarks> /// For fetching multiple pages at one time, see <see cref="PageExtensions.RefreshAsync(IEnumerable{WikiPage}, PageQueryOptions)"/>. /// </remarks> /// <exception cref="InvalidOperationException">Circular redirect detected when resolving redirects.</exception> public Task RefreshAsync(PageQueryOptions options) { return(RefreshAsync(options, new CancellationToken())); }
/// <inheritdoc cref="FetchRevisionsAsync(WikiSite,IEnumerable{int},IWikiPageQueryProvider,CancellationToken)"/> public static IAsyncEnumerable <Revision> FetchRevisionsAsync(WikiSite site, IEnumerable <int> revisionIds, PageQueryOptions options, CancellationToken cancellationToken) { return(FetchRevisionsAsync(site, revisionIds, MediaWikiHelper.QueryProviderFromOptions(options), cancellationToken)); }
/// <summary> /// Synchornously generate the sequence of pages. /// </summary> /// <param name="options">Options when querying for the pages.</param> public IEnumerable <T> EnumPages(PageQueryOptions options) { return(EnumPagesAsync(options).ToEnumerable()); }