示例#1
0
        public async Task <List <Feature> > GetAll()
        {
            var allpagesGenerator = new AllPagesGenerator(_wikiSite)
            {
                PaginationSize = 500
            };
            var results = await allpagesGenerator.EnumItemsAsync().ToListAsync().ConfigureAwait(false);

            _logger.LogInformation($"Got {results.Count} pages from iNature, fetching their content and images");
            var features = new ConcurrentBag <Feature>();
            await Task.Run(() =>
            {
                Parallel.ForEach(results, new ParallelOptions {
                    MaxDegreeOfParallelism = 5
                }, (page) =>
                {
                    var feature = PageToFeature(page).Result;
                    if (feature != null)
                    {
                        features.Add(feature);
                    }
                });
            }).ConfigureAwait(false);

            return(features.ToList());
        }
示例#2
0
        public async Task RunAsync()
        {
            var apg = new AllPagesGenerator(Site)
            {
                NamespaceId    = Site.Namespaces["Item"].Id,
                PaginationSize = 100
            };

            using (var writer = File.CreateText(FileName))
                using (var ie = apg.EnumItemsAsync().Buffer(100).GetEnumerator())
                {
                    while (await ie.MoveNext())
                    {
                        var entities = ie.Current.Select(s => new Entity(Site, WikiLink.Parse(Site, s.Title).Title)).ToList();
                        await entities.RefreshAsync(EntityQueryOptions.FetchLabels, new[]
                        {
                            "en", "zh", "zh-cn", "zh-hans"
                        });

                        foreach (var entity in entities)
                        {
                            writer.Write(entity.Id);
                            writer.Write('\t');
                            writer.Write(entity.Labels["en"]);
                            writer.Write('\t');
                            writer.Write(entity.Labels["zh-cn"] ?? entity.Labels["zh-hans"] ?? entity.Labels["zh"]);
                            writer.WriteLine();
                        }
                    }
                }
        }
        public async Task WpAllPagesGeneratorTest1()
        {
            var site      = await WpTest2SiteAsync;
            var generator = new AllPagesGenerator(site);
            var pages     = generator.EnumPages().Take(2000).ToList();

            TracePages(pages);
            AssertTitlesDistinct(pages);
        }
        public async Task WikiaAllPagesGeneratorTest()
        {
            var site      = await WikiaTestSiteAsync;
            var generator = new AllPagesGenerator(site)
            {
                NamespaceId = BuiltInNamespaces.Template
            };
            var pages = generator.EnumPages().Take(2000).ToList();

            TracePages(pages);
            AssertTitlesDistinct(pages);
        }
示例#5
0
        public async Task WpAllPagesGeneratorTest1()
        {
            var site      = await WpTest2SiteAsync;
            var generator = new AllPagesGenerator(site)
            {
                PaginationSize = 500
            };
            var pages = await generator.EnumPagesAsync().Take(2000).ToListAsync();

            TracePages(pages);
            AssertTitlesDistinct(pages);
        }
        public async Task WpAllPagesGeneratorTest2()
        {
            var site      = await WpTest2SiteAsync;
            var generator = new AllPagesGenerator(site)
            {
                StartTitle = "W", PagingSize = 20
            };
            var pages = generator.EnumPages(PageQueryOptions.FetchContent).Take(100).ToList();

            TracePages(pages);
            Assert.True(pages[0].Title[0] == 'W');
            AssertTitlesDistinct(pages);
        }
示例#7
0
        public async Task <List <Feature> > GetAll()
        {
            var allpagesGenerator = new AllPagesGenerator(_wikiSite)
            {
                PaginationSize = 500
            };
            var results = await allpagesGenerator.EnumItemsAsync().ToList().ConfigureAwait(false);

            _logger.LogInformation($"Got {results.Count} pages from iNature, fetching their content and images");
            var list     = results.AsParallel().WithDegreeOfParallelism(5).Select(PageToFeature);
            var features = await Task.WhenAll(list).ConfigureAwait(false);

            return(features.Where(f => f != null).ToList());
        }
示例#8
0
        public async Task WpTest2BulkPurgeTest()
        {
            AssertModify();
            var site = await WpTest2SiteAsync;
            // Usually 500 is the limit for normal users.
            var pages = new AllPagesGenerator(site)
            {
                PagingSize = 300
            }.EnumPages().Take(300).ToList();
            var badPage = new WikiPage(site, "Inexistent page title");

            pages.Insert(pages.Count / 2, badPage);
            Output.WriteLine("Attempt to purge: ");
            ShallowTrace(pages, 1);
            // Do a normal purge. It may take a while.
            var failedPages = await pages.PurgeAsync();

            Output.WriteLine("Failed pages: ");
            ShallowTrace(failedPages, 1);
            Assert.Equal(1, failedPages.Count);
            Assert.Same(badPage, failedPages.Single());
        }
        public async Task <IEnumerable <Event> > FetchItems(int amountOfItems)
        {
            try
            {
                var client   = new WikiClient();
                var wikiSite = new WikiSite(client, _astroWikiUrl);

                await wikiSite.Initialization;

                var allPages = new AllPagesGenerator(wikiSite);

                var provider = WikiPageQueryProvider.FromOptions(PageQueryOptions.FetchContent);
                var pages    = await allPages.EnumPagesAsync(provider).Take(amountOfItems).ToList();

                return(pages.Select(page =>
                                    Configuration.Mapper.Map <Event>(_parser.Parse(page.Content))).ToList());
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "something went wrong while fetching!");
                throw;
            }
        }
示例#10
0
        static async Task HelloWikiGenerators()
        {
            // Create a MediaWiki API client.
            var wikiClient = new WikiClient();
            // Create a MediaWiki Site instance.
            var site = new WikiSite(wikiClient, "https://en.wikipedia.org/w/api.php");
            await site.Initialization;
            // List all pages starting from item "Wiki", without redirect pages.
            var allpages = new AllPagesGenerator(site)
            {
                StartTitle      = "Wiki",
                RedirectsFilter = PropertyFilterOption.WithoutProperty
            };
            // Take the first 1000 results
            var pages = await allpages.EnumPagesAsync().Take(1000).ToList();

            foreach (var p in pages)
            {
                Console.WriteLine("{0, -30} {1, 8}B {2}", p, p.ContentLength, p.LastTouched);
            }

            // List the first 10 subcategories in Category:Cats
            Console.WriteLine();
            Console.WriteLine("Cats");
            var catmembers = new CategoryMembersGenerator(site, "Category:Cats")
            {
                MemberTypes = CategoryMemberTypes.Subcategory
            };

            pages = await catmembers.EnumPagesAsync().Take(10).ToList();

            foreach (var p in pages)
            {
                Console.WriteLine("{0, -30} {1, 8}B {2}", p, p.ContentLength, p.LastTouched);
            }
        }