public async Task WikiaAllPagesGeneratorTest() { var site = await WikiaTestSiteAsync; var generator = new AllPagesGenerator(site) { NamespaceId = BuiltInNamespaces.Template, PaginationSize = 500 }; var pages = await generator.EnumPagesAsync().Take(2000).ToList(); TracePages(pages); AssertTitlesDistinct(pages); }
public async Task WpAllPagesGeneratorTest1() { var site = await WpTest2SiteAsync; var generator = new AllPagesGenerator(site) { PaginationSize = 500 }; var pages = await generator.EnumPagesAsync().Take(2000).ToList(); TracePages(pages); AssertTitlesDistinct(pages); }
public async Task WpAllPagesGeneratorTest2() { var site = await WpTest2SiteAsync; var generator = new AllPagesGenerator(site) { StartTitle = "W", PaginationSize = 20 }; var pages = await generator.EnumPagesAsync(PageQueryOptions.FetchContent).Take(100).ToList(); TracePages(pages); Assert.True(pages[0].Title[0] == 'W'); AssertTitlesDistinct(pages); }
public async Task <IEnumerable <Event> > FetchItems(int amountOfItems) { try { var client = new WikiClient(); var wikiSite = new WikiSite(client, _astroWikiUrl); await wikiSite.Initialization; var allPages = new AllPagesGenerator(wikiSite); var provider = WikiPageQueryProvider.FromOptions(PageQueryOptions.FetchContent); var pages = await allPages.EnumPagesAsync(provider).Take(amountOfItems).ToList(); return(pages.Select(page => Configuration.Mapper.Map <Event>(_parser.Parse(page.Content))).ToList()); } catch (Exception ex) { _logger.LogError(ex, "something went wrong while fetching!"); throw; } }
static async Task HelloWikiGenerators() { // Create a MediaWiki API client. var wikiClient = new WikiClient(); // Create a MediaWiki Site instance. var site = new WikiSite(wikiClient, "https://en.wikipedia.org/w/api.php"); await site.Initialization; // List all pages starting from item "Wiki", without redirect pages. var allpages = new AllPagesGenerator(site) { StartTitle = "Wiki", RedirectsFilter = PropertyFilterOption.WithoutProperty }; // Take the first 1000 results var pages = await allpages.EnumPagesAsync().Take(1000).ToList(); foreach (var p in pages) { Console.WriteLine("{0, -30} {1, 8}B {2}", p, p.ContentLength, p.LastTouched); } // List the first 10 subcategories in Category:Cats Console.WriteLine(); Console.WriteLine("Cats"); var catmembers = new CategoryMembersGenerator(site, "Category:Cats") { MemberTypes = CategoryMemberTypes.Subcategory }; pages = await catmembers.EnumPagesAsync().Take(10).ToList(); foreach (var p in pages) { Console.WriteLine("{0, -30} {1, 8}B {2}", p, p.ContentLength, p.LastTouched); } }