public async Task <IEnumerable <Event> > FetchItems(int amountOfItems) { try { var client = new WikiClient(); var wikiSite = new WikiSite(client, _astroWikiUrl); await wikiSite.Initialization; var allPages = new AllPagesGenerator(wikiSite); var provider = WikiPageQueryProvider.FromOptions(PageQueryOptions.FetchContent); var pages = await allPages.EnumPagesAsync(provider).Take(amountOfItems).ToList(); return(pages.Select(page => Configuration.Mapper.Map <Event>(_parser.Parse(page.Content))).ToList()); } catch (Exception ex) { _logger.LogError(ex, "something went wrong while fetching!"); throw; } }
/// <inheritdoc /> protected override async Task ProcessRecordAsync(CancellationToken cancellationToken) { var pages = GetPages(); var options = PageQueryOptions.None; if (Content) { options |= PageQueryOptions.FetchContent; } if (ResolveRedirects) { options |= PageQueryOptions.ResolveRedirects; } var provider = MediaWikiHelper.QueryProviderFromOptions(options); if (Extract | GeoCoordinate) { var provider1 = WikiPageQueryProvider.FromOptions(options); if (Extract) { provider1.Properties.Add(new ExtractsPropertyProvider { AsPlainText = true, IntroductionOnly = true, MaxSentences = 1, }); } if (GeoCoordinate) { provider1.Properties.Add(new GeoCoordinatesPropertyProvider()); } provider = provider1; } await pages.RefreshAsync(provider, cancellationToken); WriteObject(pages, true); }
/// <summary> /// Gets a read-only implementation of <see cref="IWikiPageQueryProvider"/> for fetching a page. /// </summary> /// <remarks> /// This method returns a shared read-only instance for a specific <see cref="PageQueryOptions"/> value to reduce memory consumption. /// If you want to apply your customization based on the presets, use <see cref="WikiPageQueryProvider.FromOptions"/>. /// </remarks> public static IWikiPageQueryProvider QueryProviderFromOptions(PageQueryOptions options) { return(queryProviderPresets.GetOrAdd(options, k => new SealedWikiPageQueryProvider(WikiPageQueryProvider.FromOptions(options)))); }