private string GetNonExistingFilePageName(string title, string fileName)
{
var name = string.IsNullOrWhiteSpace(title) ? fileName : title;
if (Path.HasExtension(name) == false)
{
name += Path.GetExtension(fileName);
}
name = name.Replace(".jpg", ".jpeg");
var wikiFileName = "Israel_Hiking_Map_" + GetWikiName(name);
var countingFileName = Path.GetFileNameWithoutExtension(wikiFileName);
var extension = Path.GetExtension(wikiFileName);
ParallelLoopResult results;
var loopIndex = 0;
var loopRange = 5;
do
{
results = Parallel.For(loopIndex, loopIndex + loopRange, (index, options) =>
{
var pageNameToTest = GetWikiPageFileNameFromIndex(index, countingFileName, extension);
var pageToTest = new WikiPage(_site, pageNameToTest);
pageToTest.RefreshAsync(PageQueryOptions.None).Wait();
if (!pageToTest.Exists)
{
options.Break();
}
});
loopIndex += loopRange;
} while (results.LowestBreakIteration.HasValue == false);
return(GetWikiPageFileNameFromIndex(results.LowestBreakIteration.Value, countingFileName, extension));
}
public async Task BadTokenTest(string endpointUrl, string sandboxPageTitle)
{
const string invalidToken = @"INVALID_TOKEN+\";
var site = await CreateIsolatedWikiSiteAsync(endpointUrl);
var page = new WikiPage(site, sandboxPageTitle);
await page.RefreshAsync(PageQueryOptions.FetchContent);
Skip.IfNot(page.Exists, $"The page {sandboxPageTitle} doesn't exist on {site}.");
var tokensManager = typeof(WikiSite)
.GetField("tokensManager", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(site);
var tokensCache = (IDictionary <string, object>)tokensManager.GetType()
.GetField("tokensCache", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(tokensManager);
// Place an invalid token in the cache.
tokensCache["edit"] = invalidToken;
tokensCache["csrf"] = invalidToken;
Assert.Equal(invalidToken, await site.GetTokenAsync("edit"));
// This should cause token cache invalidation.
await page.UpdateContentAsync("Make an empty update.", true);
// This is a valid token
var editToken = await site.GetTokenAsync("edit");
Assert.Matches(@"^[0-9a-f]{32,}\+\\$", editToken);
}
private async Task <WikiPage> GetOrCreatePage(WikiSite site, string title)
{
if (site == null)
{
throw new ArgumentNullException(nameof(site));
}
var page = new WikiPage(site, title);
await page.RefreshAsync();
if (!page.Exists)
{
WriteOutput("Creating page: {0}", page);
page.Content = $@"<big>This is a test page for '''WikiClientLibrary'''.</big>
This page is created by an automated program for unit test purposes.
If you see this page '''OUTSIDE''' a test wiki site,
maybe you should consider viewing the history of the page,
and find out who created the page accidentally.
The original title of the page is '''{title}'''.
== See also ==
* [[Special:PrefixIndex/{title}/|Subpages]]
";
await page.UpdateContentAsync(SummaryPrefix + "Create test page for unit tests.");
}
return(page);
}
public async Task WpTest2PageReadTest1()
{
var site = await WpTest2SiteAsync;
var page = new WikiPage(site, "project:sandbox");
await page.RefreshAsync(PageQueryOptions.FetchContent);
ShallowTrace(page);
Assert.True(page.Exists);
Assert.Equal("Wikipedia:Sandbox", page.Title);
Assert.Equal(4, page.NamespaceId);
Assert.Equal("en", page.PageLanguage);
Assert.NotNull(page.Content);
// Chars vs. Bytes
Assert.True(page.Content.Length <= page.ContentLength);
Output.WriteLine(new string('-', 10));
var page2 = new WikiPage(site, 2076);
await page2.RefreshAsync();
Assert.Equal(page.PageStub, page2.PageStub);
var page3 = new WikiPage(site, "file:inexistent_file.jpg");
await page3.RefreshAsync();
ShallowTrace(page3);
Assert.False(page3.Exists);
Assert.Equal("File:Inexistent file.jpg", page3.Title);
Assert.Equal(6, page3.NamespaceId);
Assert.Equal("en", page3.PageLanguage);
}
public async Task <bool> PageExists(string title)
{
var page = new WikiPage(_site, title);
await page.RefreshAsync();
return(page.Exists);
}
public async Task <string> GetImageUrl(string pageName)
{
var imagePage = new WikiPage(_site, pageName);
await imagePage.RefreshAsync(PageQueryOptions.None);
return(imagePage.LastFileRevision?.Url);
}
public async Task WpLzhPageLanguageLinksTest()
{
var site = await WpLzhSiteAsync;
var page = new WikiPage(site, "莎拉伯恩哈特");
await page.RefreshAsync(new WikiPageQueryProvider { Properties = { new LanguageLinksPropertyProvider(LanguageLinkProperties.Autonym) } });
var langLinks = page.GetPropertyGroup <LanguageLinksPropertyGroup>()?.LanguageLinks;
ShallowTrace(langLinks);
Assert.NotNull(langLinks);
Assert.True(langLinks.Count > 120);
var langLink = langLinks.FirstOrDefault(l => l.Language == "en");
Assert.NotNull(langLink);
Assert.Equal("Sarah Bernhardt", langLink.Title);
Assert.Equal("English", langLink.Autonym);
// We didn't ask for URL so this should be null.
Assert.All(langLinks, l => Assert.Null(l.Url));
// Try out whether we still can fetch complete prop values even in the case of prop pagination.
var pages = new[] { "挪威", "坤輿", "維基共享" }.Select(t => new WikiPage(site, t)).Append(page).ToList();
await pages.RefreshAsync(new WikiPageQueryProvider { Properties = { new LanguageLinksPropertyProvider() } });
Output.WriteLine("Language links ----");
foreach (var p in pages)
{
Output.WriteLine("{0}: {1}", p, p.GetPropertyGroup <LanguageLinksPropertyGroup>()?.LanguageLinks.Count);
}
Assert.All(pages, p => Assert.True(p.GetPropertyGroup <LanguageLinksPropertyGroup>().LanguageLinks.Count > 50));
Assert.Equal(langLinks.ToDictionary(l => l.Language, l => l.Title),
page.GetPropertyGroup <LanguageLinksPropertyGroup>().LanguageLinks.ToDictionary(l => l.Language, l => l.Title));
}
private async Task <string> GetPageImageUrl(WikiPage page)
{
var match = Regex.Match(page.Content, @"תמונה=(.*)");
if (!match.Success)
{
return(null);
}
var imagePage = new WikiPage(_wikiSite, "File:" + WikimediaCommonGateway.GetWikiName(match.Groups[1].Value));
var retry = 0;
while (retry < 10 && imagePage.LastFileRevision == null)
{
try
{
await imagePage.RefreshAsync(PageQueryOptions.None).ConfigureAwait(false);
}
catch
{
retry++;
await Task.Delay(100).ConfigureAwait(false);
}
}
if (retry >= 10)
{
_logger.LogDebug("Failed to get image file for page after 10 retries: " + imagePage.Title);
}
return(imagePage.LastFileRevision?.Url);
}
private async Task <WikiPage> GetPageContent(WikiPageStub pageStub)
{
WikiPage page = null;
var retry = 0;
while (retry < 10 && page?.Content == null)
{
try
{
page = new WikiPage(_wikiSite, pageStub.Id);
await page.RefreshAsync(PageQueryOptions.FetchContent).ConfigureAwait(false);
}
catch
{
retry++;
await Task.Delay(100).ConfigureAwait(false);
}
}
if (retry >= 10)
{
_logger.LogDebug("Failed to get content for page after 10 retries: " + pageStub.Title);
}
return(page);
}
private IEnumerable <WikiPage> GetStoryLinksPageList(WikiSite site, string pageTitle)
{
var targetPage = new WikiPage(site, pageTitle);
targetPage.RefreshAsync(PageQueryOptions.FetchContent | PageQueryOptions.ResolveRedirects).Wait(); //Load page content
//Get page text
var parser = new WikitextParser();
var wikiPageText = parser.Parse(targetPage.Content);
IEnumerable <Template> templateList = new List <Template>();
var header = wikiPageText.Lines.SelectMany(x => x.EnumDescendants().OfType <Heading>()).Where(y => HeadersToSearch.Contains(y.ToPlainText())).SingleOrDefault();
if (header != null)
{
templateList = header.EnumDescendants().OfType <Template>();
}
else
{
templateList = wikiPageText.EnumDescendants().OfType <Template>();
}
var storyLinkTemplates = templateList.Where(template => template.Name.Equals("storylink"));
return(storyLinkTemplates.Select(template => new WikiPage(site, template.Arguments.Single().Value.ToPlainText())));
}
public async Task LocalFileUploadTest1(string fileName, string imageName)
{
const string ReuploadSuffix = "\n\nReuploaded.";
var site = await SiteAsync;
var file = Utility.GetDemoImage(imageName);
var result = await site.UploadAsync(fileName, new StreamUploadSource(file.ContentStream), file.Description, false);
// Usually we should notify the user, then perform the re-upload ignoring the warning.
try
{
if (result.Warnings.TitleExists)
{
result = await site.UploadAsync(fileName,
new FileKeyUploadSource(result.FileKey), file.Description + ReuploadSuffix, true);
}
Assert.Equal(UploadResultCode.Success, result.ResultCode);
}
catch (OperationFailedException ex) when(ex.ErrorCode == "fileexists-no-change")
{
Output.WriteLine(ex.Message);
}
ShallowTrace(result);
var page = new WikiPage(site, fileName, BuiltInNamespaces.File);
await page.RefreshAsync();
ShallowTrace(page);
Assert.True(page.Exists);
Assert.Equal(file.Sha1, page.LastFileRevision.Sha1.ToUpperInvariant());
}
public async Task WpLzhPageReadDisambigTest()
{
var site = await WpLzhSiteAsync;
var page = new WikiPage(site, "中國_(釋義)");
await page.RefreshAsync();
Assert.True(await page.IsDisambiguationAsync());
}
public async Task WikiaPageReadDisambigTest()
{
var site = await WikiaTestSiteAsync;
var page = new WikiPage(site, "Test (Disambiguation)");
await page.RefreshAsync();
Assert.True(await page.IsDisambiguationAsync());
}
public async Task WpLzhRedirectedPageReadTest()
{
var site = await WpLzhSiteAsync;
var page = new WikiPage(site, "project:sandbox");
await page.RefreshAsync(PageQueryOptions.ResolveRedirects);
Assert.Equal("維基大典:沙盒", page.Title);
Assert.Equal(4, page.NamespaceId);
ShallowTrace(page);
}
public async Task WikiaPageReadByIdTest()
{
var site = await WikiaTestSiteAsync;
var page = new WikiPage(site, 190273);
await page.RefreshAsync();
Assert.Equal("Mediawiki 1.19 test Wiki:Sandbox", page.Title);
Assert.Equal(4, page.NamespaceId);
ShallowTrace(page);
}
public async Task WikiaPageReadTest()
{
var site = await WikiaTestSiteAsync;
var page = new WikiPage(site, "Project:Sandbox");
await page.RefreshAsync(PageQueryOptions.FetchContent);
Assert.Equal("Mediawiki 1.19 test Wiki:Sandbox", page.Title);
Assert.Equal(4, page.NamespaceId);
ShallowTrace(page);
}
private async Task <Feature> ConvertPageToFeature(WikiPage page, string language)
{
await page.RefreshAsync(new WikiPageQueryProvider
{
Properties =
{
new ExtractsPropertyProvider {
AsPlainText = true, IntroductionOnly = true, MaxSentences = 1
},
new PageImagesPropertyProvider {
QueryOriginalImage = true
},
new GeoCoordinatesPropertyProvider {
QueryPrimaryCoordinate = true
},
new RevisionsPropertyProvider {
FetchContent = false
}
}
});
if (page.Exists == false)
{
return(null);
}
var geoCoordinate = page.GetPropertyGroup <GeoCoordinatesPropertyGroup>().PrimaryCoordinate;
if (geoCoordinate.IsEmpty)
{
return(null);
}
var coordinate = new CoordinateZ(geoCoordinate.Longitude, geoCoordinate.Latitude, double.NaN);
var attributes = GetAttributes(coordinate, page.Title, page.Id.ToString(), language);
attributes.Add(FeatureAttributes.DESCRIPTION + ":" + language, page.GetPropertyGroup <ExtractsPropertyGroup>().Extract ?? string.Empty);
var imageUrl = page.GetPropertyGroup <PageImagesPropertyGroup>().OriginalImage.Url ?? string.Empty;
if (!string.IsNullOrWhiteSpace(imageUrl) &&
(imageUrl.EndsWith(".jpg", StringComparison.OrdinalIgnoreCase) ||
imageUrl.EndsWith(".jpeg", StringComparison.OrdinalIgnoreCase) ||
imageUrl.EndsWith(".png", StringComparison.OrdinalIgnoreCase) ||
imageUrl.EndsWith(".bmp", StringComparison.OrdinalIgnoreCase)))
{
attributes.Add(FeatureAttributes.IMAGE_URL, imageUrl);
}
attributes.Add(FeatureAttributes.POI_USER_NAME + ":" + language, page.LastRevision.UserName);
attributes.Add(FeatureAttributes.POI_USER_ADDRESS + ":" + language, _wikiSites[language].SiteInfo.MakeArticleUrl($"User:{Uri.EscapeUriString(page.LastRevision.UserName)}"));
attributes.Add(FeatureAttributes.POI_LAST_MODIFIED + ":" + language, page.LastRevision.TimeStamp.ToString("o"));
var feature = new Feature(new Point(coordinate), attributes);
feature.SetTitles();
feature.SetId();
return(feature);
}
public async Task WpLzhFetchFileTest()
{
var site = await WpLzhSiteAsync;
var file = new WikiPage(site, "File:Empress Suiko.jpg");
await file.RefreshAsync();
ShallowTrace(file);
//Assert.True(file.Exists); //It's on WikiMedia!
Assert.Equal(58865, file.LastFileRevision.Size);
Assert.Equal("7aa12c613c156dd125212d85a072b250625ae39f", file.LastFileRevision.Sha1.ToLowerInvariant());
}
public async Task WpTest2PageWriteTest1()
{
AssertModify();
var site = await WpTest2SiteAsync;
var page = new WikiPage(site, "project:sandbox");
await page.RefreshAsync(PageQueryOptions.FetchContent);
page.Content += "\n\nTest from WikiClientLibrary.";
Output.WriteLine(page.Content);
await page.UpdateContentAsync(SummaryPrefix + "Edit sandbox page.");
}
public async Task WpTest2PageWriteTest3()
{
AssertModify();
var site = await WpTest2SiteAsync;
var page = new WikiPage(site, "Special:RecentChanges");
await page.RefreshAsync(PageQueryOptions.FetchContent);
Assert.True(page.IsSpecialPage);
page.Content += "\n\nTest from WikiClientLibrary.";
await Assert.ThrowsAsync <InvalidOperationException>(() =>
page.UpdateContentAsync(SummaryPrefix + "Attempt to edit a special page."));
}
public async Task FetchRelatedPagesTest()
{
var site = await WikiaTestSiteAsync;
var mainPage = new WikiPage(site, site.SiteInfo.MainPage);
await mainPage.RefreshAsync();
var relatedPages = await site.FetchRelatedPagesAsync(mainPage.Id);
ShallowTrace(relatedPages);
// These are just random titles.
Assert.All(relatedPages, p => Assert.Matches(@"\w+\d+", p.Title));
}
public async Task WpTest2PageWriteTest2()
{
AssertModify();
var site = await WpTest2SiteAsync;
var page = new WikiPage(site, "Test page");
await page.RefreshAsync(PageQueryOptions.FetchContent);
Assert.True(page.Protections.Any(), "To perform this test, the working page should be protected.");
page.Content += "\n\nTest from WikiClientLibrary.";
await Assert.ThrowsAsync <UnauthorizedOperationException>(() =>
page.UpdateContentAsync(SummaryPrefix + "Attempt to edit a protected page."));
}
public async Task WpTest2PageReadRedirectTest()
{
var site = await WpTest2SiteAsync;
var page = new WikiPage(site, "Foo");
await page.RefreshAsync();
Assert.True(page.IsRedirect);
var target = await page.GetRedirectTargetAsync();
ShallowTrace(target);
Assert.Equal("Foo24", target.Title);
Assert.True(target.RedirectPath.SequenceEqual(new[] { "Foo", "Foo2", "Foo23" }));
}
private async Task <FeatureCollection> ConvertPageToFeatureCollection(WikiPage page, string language)
{
await page.RefreshAsync(new WikiPageQueryProvider
{
Properties =
{
new ExtractsPropertyProvider {
AsPlainText = true, IntroductionOnly = true, MaxSentences = 1
},
new PageImagesPropertyProvider {
QueryOriginalImage = true
},
new GeoCoordinatesPropertyProvider {
QueryPrimaryCoordinate = true
},
new RevisionsPropertyProvider {
FetchContent = false
}
}
});
if (page.Exists == false)
{
return(null);
}
var geoCoordinate = page.GetPropertyGroup <GeoCoordinatesPropertyGroup>().PrimaryCoordinate;
if (geoCoordinate.IsEmpty)
{
return(null);
}
var coordinate = new Coordinate(geoCoordinate.Longitude, geoCoordinate.Latitude);
var attributes = GetAttributes(coordinate, page.Title, page.Id.ToString(), language);
attributes.Add(FeatureAttributes.DESCRIPTION + ":" + language, page.GetPropertyGroup <ExtractsPropertyGroup>().Extract ?? string.Empty);
var imageUrl = page.GetPropertyGroup <PageImagesPropertyGroup>().OriginalImage.Url ?? string.Empty;
attributes.Add(FeatureAttributes.IMAGE_URL, imageUrl.EndsWith(".svg") ? string.Empty : imageUrl);
attributes.Add(FeatureAttributes.POI_USER_NAME, page.LastRevision.UserName);
attributes.Add(FeatureAttributes.POI_USER_ADDRESS, _wikiSites[language].SiteInfo.MakeArticleUrl($"User:{Uri.EscapeUriString(page.LastRevision.UserName)}"));
attributes.Add(FeatureAttributes.POI_LAST_MODIFIED, page.LastRevision.TimeStamp.ToString("o"));
return(new FeatureCollection(new Collection <IFeature> {
new Feature(new Point(coordinate), attributes)
}));
}
/// <summary>
/// Get normalised page title from API.
/// </summary>
/// <param name="title">Page title.</param>
/// <param name="url">URL string in <code>https://ru.wikipedia.org/wiki/$1</code> format.</param>
public static async Task <string> GetNormalisedTitle(string title, string url)
{
string pageTitle = null;
string wikiUrlPattern = "/wiki/$1";
string apiUrl = url.Replace(wikiUrlPattern, "/w/api.php");
WikiSite site = null;
bool siteWasInitialised = WikiSiteInfo.ContainsKey(url);
if (siteWasInitialised)
{
site = WikiSiteInfo[url];
}
else
{
site = new WikiSite(Program.WikiClient, apiUrl);
}
try
{
if (!siteWasInitialised)
{
await site.Initialization;
}
var page = new WikiPage(site, title);
await page.RefreshAsync();
pageTitle = page.Title;
}
catch (Exception ex)
{
Program.LogMessage($"Wiki ({url}) can’t be reached: {ex.InnerException}", "Linking", "warning");
}
// Restore the anchor from original title
if (title.Contains('#'))
{
pageTitle += "#" + EncodePageTitle(title.Split('#')?[1], false);
}
await Task.CompletedTask;
return(pageTitle);
}
public async Task WpLzhFetchFileWithExtMetadataTest()
{
var site = await WpLzhSiteAsync;
var file = new WikiPage(site, "File:Empress Suiko.jpg");
await file.RefreshAsync(new WikiPageQueryProvider
{
Properties =
{
new FileInfoPropertyProvider {
QueryExtMetadata = true
}
}
});
ShallowTrace(file);
Assert.NotNull(file.LastFileRevision.ExtMetadata);
Assert.Equal("mediawiki-metadata", file.LastFileRevision.ExtMetadata["DateTime"]?.Source);
}
public async Task WikiaCategoryMembersGeneratorTest()
{
var site = await WikiaTestSiteAsync;
var cat = new WikiPage(site, "Category:BlogListingPage");
await cat.RefreshAsync();
Output.WriteLine(cat.ToString());
var generator = new CategoryMembersGenerator(cat)
{
PaginationSize = 50
};
var pages = await generator.EnumPagesAsync().ToList();
TracePages(pages);
AssertTitlesDistinct(pages);
var catInfo = cat.GetPropertyGroup <CategoryInfoPropertyGroup>();
Assert.Equal(catInfo.MembersCount, pages.Count);
}
public async Task WpCategoryMembersGeneratorTest()
{
var site = await WpTest2SiteAsync;
var cat = new WikiPage(site, "Category:Template documentation pages");
await cat.RefreshAsync();
WriteOutput(cat);
var generator = new CategoryMembersGenerator(cat)
{
PaginationSize = 50
};
var pages = await generator.EnumPagesAsync().ToListAsync();
TracePages(pages);
AssertTitlesDistinct(pages);
var catInfo = cat.GetPropertyGroup <CategoryInfoPropertyGroup>();
Assert.Equal(catInfo.MembersCount, pages.Count);
}
public async Task <List <WikiJobRequest> > GetNewJobDefinitions()
{
IEnumerable <WikiJobRequest> jobs = null;
using (var client = new WikiClient
{
ClientUserAgent = "WCLQuickStart/1.0 (your user name or contact information here)"
})
{
try
{
_log.Information($"Logging into Wiki");
var site = _wikiAccessLogic.GetLoggedInWikiSite(_wikiLoginConfig, client, _log);
var page = new WikiPage(site, _wikiRequestPage);
_log.Information($"Fetching content from job request page {page.Title}");
// Fetch content from the job request page so we can build jobs from it
await page.RefreshAsync(PageQueryOptions.FetchContent
| PageQueryOptions.ResolveRedirects);
if ((page?.Content ?? "").Length > 1)
{
var ast = new WikitextParser().Parse(page?.Content);
var templates = ast.Lines.First().EnumDescendants().OfType <Template>();
_log.Information("Building jobs.");
jobs = templates.Select(template => WikiJobRequestFactory.GetWikiJobRequest((JobType)Enum.Parse(typeof(JobType), template.Arguments.Single(arg => arg.Name.ToPlainText() == "type").Value.ToPlainText().Replace(" ", "") + "Job"), GetTimeZone(), template));
foreach (WikiJobRequest request in jobs)
{
_log.Information($"Request {request.RawRequest} has status {request.Status}");
}
}
await site.LogoutAsync();
}
catch (Exception ex)
{
_log.Error(ex, "An error occurred while trying to fetch new requests: ");
}
}
return(jobs?.ToList());
}
public static async Task AddToWiki(string URL, string username, string password, string pagename, string content)
{
WikiClient wikiClient = new WikiClient()
{
ClientUserAgent = "CRMViewer/1.0"
};
WikiSite wikiSite = new WikiSite(wikiClient, URL);
await wikiSite.Initialization;
CancellationToken cancellationToken = new CancellationToken();
await wikiSite.LoginAsync(username, password);
WikiPage wikiPage = new WikiPage(wikiSite, pagename);
await wikiPage.RefreshAsync();
wikiPage.Content = content;
await wikiPage.UpdateContentAsync("auto update", true, true);
//await wikiSite.LogoutAsync();
}