public async Task <List <Feature> > GetAll() { var allpagesGenerator = new AllPagesGenerator(_wikiSite) { PaginationSize = 500 }; var results = await allpagesGenerator.EnumItemsAsync().ToListAsync().ConfigureAwait(false); _logger.LogInformation($"Got {results.Count} pages from iNature, fetching their content and images"); var features = new ConcurrentBag <Feature>(); await Task.Run(() => { Parallel.ForEach(results, new ParallelOptions { MaxDegreeOfParallelism = 5 }, (page) => { var feature = PageToFeature(page).Result; if (feature != null) { features.Add(feature); } }); }).ConfigureAwait(false); return(features.ToList()); }
public async Task RunAsync() { var apg = new AllPagesGenerator(Site) { NamespaceId = Site.Namespaces["Item"].Id, PaginationSize = 100 }; using (var writer = File.CreateText(FileName)) using (var ie = apg.EnumItemsAsync().Buffer(100).GetEnumerator()) { while (await ie.MoveNext()) { var entities = ie.Current.Select(s => new Entity(Site, WikiLink.Parse(Site, s.Title).Title)).ToList(); await entities.RefreshAsync(EntityQueryOptions.FetchLabels, new[] { "en", "zh", "zh-cn", "zh-hans" }); foreach (var entity in entities) { writer.Write(entity.Id); writer.Write('\t'); writer.Write(entity.Labels["en"]); writer.Write('\t'); writer.Write(entity.Labels["zh-cn"] ?? entity.Labels["zh-hans"] ?? entity.Labels["zh"]); writer.WriteLine(); } } } }
public async Task WpAllPagesGeneratorTest1() { var site = await WpTest2SiteAsync; var generator = new AllPagesGenerator(site); var pages = generator.EnumPages().Take(2000).ToList(); TracePages(pages); AssertTitlesDistinct(pages); }
public async Task WikiaAllPagesGeneratorTest() { var site = await WikiaTestSiteAsync; var generator = new AllPagesGenerator(site) { NamespaceId = BuiltInNamespaces.Template }; var pages = generator.EnumPages().Take(2000).ToList(); TracePages(pages); AssertTitlesDistinct(pages); }
public async Task WpAllPagesGeneratorTest1() { var site = await WpTest2SiteAsync; var generator = new AllPagesGenerator(site) { PaginationSize = 500 }; var pages = await generator.EnumPagesAsync().Take(2000).ToListAsync(); TracePages(pages); AssertTitlesDistinct(pages); }
public async Task WpAllPagesGeneratorTest2() { var site = await WpTest2SiteAsync; var generator = new AllPagesGenerator(site) { StartTitle = "W", PagingSize = 20 }; var pages = generator.EnumPages(PageQueryOptions.FetchContent).Take(100).ToList(); TracePages(pages); Assert.True(pages[0].Title[0] == 'W'); AssertTitlesDistinct(pages); }
public async Task <List <Feature> > GetAll() { var allpagesGenerator = new AllPagesGenerator(_wikiSite) { PaginationSize = 500 }; var results = await allpagesGenerator.EnumItemsAsync().ToList().ConfigureAwait(false); _logger.LogInformation($"Got {results.Count} pages from iNature, fetching their content and images"); var list = results.AsParallel().WithDegreeOfParallelism(5).Select(PageToFeature); var features = await Task.WhenAll(list).ConfigureAwait(false); return(features.Where(f => f != null).ToList()); }
public async Task WpTest2BulkPurgeTest() { AssertModify(); var site = await WpTest2SiteAsync; // Usually 500 is the limit for normal users. var pages = new AllPagesGenerator(site) { PagingSize = 300 }.EnumPages().Take(300).ToList(); var badPage = new WikiPage(site, "Inexistent page title"); pages.Insert(pages.Count / 2, badPage); Output.WriteLine("Attempt to purge: "); ShallowTrace(pages, 1); // Do a normal purge. It may take a while. var failedPages = await pages.PurgeAsync(); Output.WriteLine("Failed pages: "); ShallowTrace(failedPages, 1); Assert.Equal(1, failedPages.Count); Assert.Same(badPage, failedPages.Single()); }
public async Task <IEnumerable <Event> > FetchItems(int amountOfItems) { try { var client = new WikiClient(); var wikiSite = new WikiSite(client, _astroWikiUrl); await wikiSite.Initialization; var allPages = new AllPagesGenerator(wikiSite); var provider = WikiPageQueryProvider.FromOptions(PageQueryOptions.FetchContent); var pages = await allPages.EnumPagesAsync(provider).Take(amountOfItems).ToList(); return(pages.Select(page => Configuration.Mapper.Map <Event>(_parser.Parse(page.Content))).ToList()); } catch (Exception ex) { _logger.LogError(ex, "something went wrong while fetching!"); throw; } }
static async Task HelloWikiGenerators() { // Create a MediaWiki API client. var wikiClient = new WikiClient(); // Create a MediaWiki Site instance. var site = new WikiSite(wikiClient, "https://en.wikipedia.org/w/api.php"); await site.Initialization; // List all pages starting from item "Wiki", without redirect pages. var allpages = new AllPagesGenerator(site) { StartTitle = "Wiki", RedirectsFilter = PropertyFilterOption.WithoutProperty }; // Take the first 1000 results var pages = await allpages.EnumPagesAsync().Take(1000).ToList(); foreach (var p in pages) { Console.WriteLine("{0, -30} {1, 8}B {2}", p, p.ContentLength, p.LastTouched); } // List the first 10 subcategories in Category:Cats Console.WriteLine(); Console.WriteLine("Cats"); var catmembers = new CategoryMembersGenerator(site, "Category:Cats") { MemberTypes = CategoryMemberTypes.Subcategory }; pages = await catmembers.EnumPagesAsync().Take(10).ToList(); foreach (var p in pages) { Console.WriteLine("{0, -30} {1, 8}B {2}", p, p.ContentLength, p.LastTouched); } }