public async Task <List <Feature> > GetAll()
{
var allpagesGenerator = new AllPagesGenerator(_wikiSite)
{
PaginationSize = 500
};
var results = await allpagesGenerator.EnumItemsAsync().ToListAsync().ConfigureAwait(false);
_logger.LogInformation($"Got {results.Count} pages from iNature, fetching their content and images");
var features = new ConcurrentBag <Feature>();
await Task.Run(() =>
{
Parallel.ForEach(results, new ParallelOptions {
MaxDegreeOfParallelism = 5
}, (page) =>
{
var feature = PageToFeature(page).Result;
if (feature != null)
{
features.Add(feature);
}
});
}).ConfigureAwait(false);
return(features.ToList());
}
public async Task RunAsync()
{
var apg = new AllPagesGenerator(Site)
{
NamespaceId = Site.Namespaces["Item"].Id,
PaginationSize = 100
};
using (var writer = File.CreateText(FileName))
using (var ie = apg.EnumItemsAsync().Buffer(100).GetEnumerator())
{
while (await ie.MoveNext())
{
var entities = ie.Current.Select(s => new Entity(Site, WikiLink.Parse(Site, s.Title).Title)).ToList();
await entities.RefreshAsync(EntityQueryOptions.FetchLabels, new[]
{
"en", "zh", "zh-cn", "zh-hans"
});
foreach (var entity in entities)
{
writer.Write(entity.Id);
writer.Write('\t');
writer.Write(entity.Labels["en"]);
writer.Write('\t');
writer.Write(entity.Labels["zh-cn"] ?? entity.Labels["zh-hans"] ?? entity.Labels["zh"]);
writer.WriteLine();
}
}
}
}
public async Task WpAllPagesGeneratorTest1()
{
var site = await WpTest2SiteAsync;
var generator = new AllPagesGenerator(site);
var pages = generator.EnumPages().Take(2000).ToList();
TracePages(pages);
AssertTitlesDistinct(pages);
}
public async Task WikiaAllPagesGeneratorTest()
{
var site = await WikiaTestSiteAsync;
var generator = new AllPagesGenerator(site)
{
NamespaceId = BuiltInNamespaces.Template
};
var pages = generator.EnumPages().Take(2000).ToList();
TracePages(pages);
AssertTitlesDistinct(pages);
}
public async Task WpAllPagesGeneratorTest1()
{
var site = await WpTest2SiteAsync;
var generator = new AllPagesGenerator(site)
{
PaginationSize = 500
};
var pages = await generator.EnumPagesAsync().Take(2000).ToListAsync();
TracePages(pages);
AssertTitlesDistinct(pages);
}
public async Task WpAllPagesGeneratorTest2()
{
var site = await WpTest2SiteAsync;
var generator = new AllPagesGenerator(site)
{
StartTitle = "W", PagingSize = 20
};
var pages = generator.EnumPages(PageQueryOptions.FetchContent).Take(100).ToList();
TracePages(pages);
Assert.True(pages[0].Title[0] == 'W');
AssertTitlesDistinct(pages);
}
public async Task <List <Feature> > GetAll()
{
var allpagesGenerator = new AllPagesGenerator(_wikiSite)
{
PaginationSize = 500
};
var results = await allpagesGenerator.EnumItemsAsync().ToList().ConfigureAwait(false);
_logger.LogInformation($"Got {results.Count} pages from iNature, fetching their content and images");
var list = results.AsParallel().WithDegreeOfParallelism(5).Select(PageToFeature);
var features = await Task.WhenAll(list).ConfigureAwait(false);
return(features.Where(f => f != null).ToList());
}
public async Task WpTest2BulkPurgeTest()
{
AssertModify();
var site = await WpTest2SiteAsync;
// Usually 500 is the limit for normal users.
var pages = new AllPagesGenerator(site)
{
PagingSize = 300
}.EnumPages().Take(300).ToList();
var badPage = new WikiPage(site, "Inexistent page title");
pages.Insert(pages.Count / 2, badPage);
Output.WriteLine("Attempt to purge: ");
ShallowTrace(pages, 1);
// Do a normal purge. It may take a while.
var failedPages = await pages.PurgeAsync();
Output.WriteLine("Failed pages: ");
ShallowTrace(failedPages, 1);
Assert.Equal(1, failedPages.Count);
Assert.Same(badPage, failedPages.Single());
}
public async Task <IEnumerable <Event> > FetchItems(int amountOfItems)
{
try
{
var client = new WikiClient();
var wikiSite = new WikiSite(client, _astroWikiUrl);
await wikiSite.Initialization;
var allPages = new AllPagesGenerator(wikiSite);
var provider = WikiPageQueryProvider.FromOptions(PageQueryOptions.FetchContent);
var pages = await allPages.EnumPagesAsync(provider).Take(amountOfItems).ToList();
return(pages.Select(page =>
Configuration.Mapper.Map <Event>(_parser.Parse(page.Content))).ToList());
}
catch (Exception ex)
{
_logger.LogError(ex, "something went wrong while fetching!");
throw;
}
}
static async Task HelloWikiGenerators()
{
// Create a MediaWiki API client.
var wikiClient = new WikiClient();
// Create a MediaWiki Site instance.
var site = new WikiSite(wikiClient, "https://en.wikipedia.org/w/api.php");
await site.Initialization;
// List all pages starting from item "Wiki", without redirect pages.
var allpages = new AllPagesGenerator(site)
{
StartTitle = "Wiki",
RedirectsFilter = PropertyFilterOption.WithoutProperty
};
// Take the first 1000 results
var pages = await allpages.EnumPagesAsync().Take(1000).ToList();
foreach (var p in pages)
{
Console.WriteLine("{0, -30} {1, 8}B {2}", p, p.ContentLength, p.LastTouched);
}
// List the first 10 subcategories in Category:Cats
Console.WriteLine();
Console.WriteLine("Cats");
var catmembers = new CategoryMembersGenerator(site, "Category:Cats")
{
MemberTypes = CategoryMemberTypes.Subcategory
};
pages = await catmembers.EnumPagesAsync().Take(10).ToList();
foreach (var p in pages)
{
Console.WriteLine("{0, -30} {1, 8}B {2}", p, p.ContentLength, p.LastTouched);
}
}
