private static async Task <bool> _editPageAsync(MediaWikiClient client, EditHistory history, string pageTitle, string pageContent)
{
// Check to see if we've made this edit before.
// If we've already made this page before, don't do anything.
EditRecord record = await history.GetEditRecordAsync(pageTitle, pageContent);
if (record is null)
{
// Get existing page content.
// This allows us to make sure that no one has removed the "{{BotGenerated}}" flag.
// If it has been removed, do not modify the page.
MediaWikiApiParseRequestResult parse_result = client.Parse(pageTitle, new ParseParameters());
if (parse_result.ErrorCode == ErrorCode.MissingTitle || parse_result.Text.Contains(BotFlag))
{
if (parse_result.ErrorCode == ErrorCode.MissingTitle)
{
_log(string.Format("creating page \"{0}\"", pageTitle));
}
else
{
_log(string.Format("editing page \"{0}\"", pageTitle));
}
try {
client.Edit(pageTitle, new EditParameters {
Action = EditAction.Text,
Text = pageContent
});
// Make a record of the edit.
await history.AddEditRecordAsync(pageTitle, pageContent);
// Return true to indicate that edits have occurred.
return(true);
}
catch (Exception ex) {
_log(ex.ToString());
}
}
else
{
_log(string.Format("skipping page \"{0}\" (manually edited)", pageTitle));
}
}
else
{
_log(string.Format("skipping page \"{0}\" (previously edited)", pageTitle));
}
// Return false to indicate that no edits have occurred.
return(false);
}
private static async Task _editSpeciesPageAsync(MediaWikiClient client, EditHistory history, Species species, string pageTitle, string pageContent)
{
if (await _editPageAsync(client, history, pageTitle, pageContent))
{
// If the edit was successful, associated it with this species.
EditRecord record = await history.GetEditRecordAsync(pageTitle, pageContent);
if (record != null)
{
await history.AddEditRecordAsync(species.Id, record);
// Because it's possible that the species was renamed, we need to look at past edits to find previous titles of the same page.
// Old pages for renamed species will be deleted.
EditRecord[] edit_records = (await history.GetEditRecordsAsync(species.Id))
.Where(x => x.Id != record.Id && x.Title.ToLower() != record.Title.ToLower())
.ToArray();
// Delete all created pages where the old title does not match the current title.
foreach (EditRecord i in edit_records)
{
MediaWikiApiParseRequestResult parse_result = client.Parse(i.Title, new ParseParameters());
if (parse_result.Text.Contains(BotFlag))
{
// Only delete pages that haven't been manually edited.
client.Delete(i.Title, new DeleteParameters {
Reason = "species page moved to " + pageTitle
});
// Add an edit record for this page so that we can restore the content later without it thinking we've already made this edit.
// This is important, because this step can delete redirects when a page with redirects is updated. By creating a new edit record, the redirect will be recreated.
await history.AddEditRecordAsync(i.Title, string.Empty);
}
}
// We also need to delete any redirect pages that are now invalid (i.e. when specific epithet that points to a common name is changed).
// Delete all redirects that point to this page (or one of this page's previous titles).
RedirectRecord[] redirect_records = (await history.GetRedirectRecordsAsync())
.Where(i => i.Target == pageTitle || edit_records.Any(j => j.Title == i.Target)) // points to the title of this page, or one of its previous titles
.Where(i => i.Title != species.FullName) // the title doesn't match this species' full name (the species has been renamed)
.ToArray();
foreach (RedirectRecord j in redirect_records)
{
MediaWikiApiParseRequestResult parse_result = client.Parse(j.Title, new ParseParameters());
if (parse_result.IsRedirect && parse_result.Text.Contains(BotFlag))
{
// Only delete pages that haven't been manually edited.
client.Delete(j.Title, new DeleteParameters {
Reason = "outdated redirect"
});
}
}
}
}
}
private static async Task <string> UploadPictureAsync(MediaWikiClient client, EditHistory history, UploadParameters parameters, bool allowRetry)
{
// Check if we've already uploaded this file before.
// If we've uploaded it before, return the filename that we uploaded it with.
UploadRecord record = await history.GetUploadRecordAsync(parameters.FilePath);
if (record is null)
{
// Get page for the file and check for the bot flag.
// This prevents us from overwriting images that users uploaded manually.
MediaWikiApiParseRequestResult page_content = client.Parse(parameters.PageTitle, new ParseParameters());
if (page_content.ErrorCode == ErrorCode.MissingTitle || page_content.Text.Contains(BotFlag))
{
// Attempt to upload the file.
try {
parameters.Text = BotFlag + '\n' + parameters.Text;
MediaWikiApiRequestResult result = client.Upload(parameters);
if (!result.Success)
{
_log(result.ErrorMessage);
}
if (result.ErrorCode == ErrorCode.VerificationError && allowRetry)
{
// This means that the file extension didn't match (e.g., filename has ".png" when the file format is actually ".jpg").
// Try changing the file extension and reuploading, because sometimes URLs stored in the bot will have this problem.
string ext = System.IO.Path.GetExtension(parameters.UploadFileName);
ext = (ext == ".png") ? ".jpg" : ".png";
parameters.UploadFileName = System.IO.Path.ChangeExtension(parameters.UploadFileName, ext);
_log("file extension didn't match, retrying upload");
return(await UploadPictureAsync(client, history, parameters, false));
}
else
{
if (result.Success || result.ErrorCode == ErrorCode.FileExistsNoChange)
{
// If the upload succeeded, record the file upload so that we can skip it in the future.
await history.AddUploadRecordAsync(parameters.FilePath, parameters.UploadFileName);
// Add the bot flag to the page content.
// client.Edit(parameters.PageTitle, new EditParameters { Text = BotFlag });
}
else
{
parameters.UploadFileName = string.Empty;
}
}
}
catch (Exception ex) {
parameters.UploadFileName = string.Empty;
_log(ex.ToString());
}
}
else
{
_log(string.Format("skipping file \"{0}\" (manually edited)", parameters.UploadFileName));
}
}
else
{
// This image has been uploaded previously, so just return its path.
_log(string.Format("skipping file \"{0}\" (previously uploaded)", parameters.UploadFileName));
parameters.UploadFileName = record.UploadFileName;
}
return(parameters.UploadFileName);
}
private static async Task <string[]> UploadSpeciesGalleryAsync(MediaWikiClient client, EditHistory history, Species species)
{
// Upload all images in the given species' gallery.
Picture[] pictures = await SpeciesUtils.GetPicturesAsync(species);
List <string> uploadedFilenames = new List <string>();
if (pictures != null)
{
foreach (Picture picture in pictures)
{
// Skip the image if it's the same as the species' default image, because we would've already uploaded it
if (picture.url == species.Picture)
{
continue;
}
string uploadedFilename = GeneratePictureFilenameFromSpecies(species, picture.url);
if (!string.IsNullOrEmpty(uploadedFilename))
{
UploadParameters uploadParameters = new UploadParameters {
UploadFileName = uploadedFilename,
FilePath = picture.url,
Text = picture.description
};
uploadedFilename = await UploadPictureAsync(client, history, uploadParameters, true);
if (!string.IsNullOrEmpty(uploadedFilename))
{
uploadedFilenames.Add(uploadedFilename);
}
}
else
{
_log(string.Format("Failed to generate filename for picture: {0}", picture.url));
}
}
}
return(uploadedFilenames.ToArray());
}
private static async Task <string> UploadSpeciesPictureAsync(MediaWikiClient client, EditHistory history, Species species)
{
// Generate a filename for the image, which will be the filename when it's uploaded to the wiki.
string uploadedFilename = GeneratePictureFilenameFromSpecies(species);
if (!string.IsNullOrEmpty(uploadedFilename))
{
// Attempt to upload the image.
UploadParameters uploadParameters = new UploadParameters {
UploadFileName = uploadedFilename,
FilePath = species.Picture
};
return(await UploadPictureAsync(client, history, uploadParameters, true));
}
return(string.Empty);
}
public async Task MainAsync(string[] args)
{
_log("loading configuration");
Config config = JsonConvert.DeserializeObject <Config>(System.IO.File.ReadAllText("wikibot-config.json"));
_log("initializing mediawiki client");
MediaWikiClient client = new MediaWikiClient {
Protocol = config.Protocol,
Server = config.Server,
ApiPath = config.ApiPath,
UserAgent = config.UserAgent
};
client.Log += _log;
EditHistory history = new EditHistory();
if (client.Login(config.Username, config.Password).Success)
{
_log("generating link dictionary");
WikiLinkList LinkifyList = await _generateLinkifyListAsync();
_log("synchronizing species");
_log("getting species from database");
Species[] speciesList = await SpeciesUtils.GetSpeciesAsync();
_log(string.Format("got {0} results", speciesList.Count()));
foreach (Species species in speciesList)
{
_log(string.Format("synchronizing species {0}", species.ShortName));
// Create the page builder.
SpeciesPageBuilder pageBuilder = new SpeciesPageBuilder(species, WikiPageTemplate.Open(SpeciesTemplateFilePath))
{
AllSpecies = speciesList,
LinkList = LinkifyList
};
// Attempt to upload the species' picture.
pageBuilder.PictureFilenames.Add(await UploadSpeciesPictureAsync(client, history, species));
pageBuilder.PictureFilenames.AddRange(await UploadSpeciesGalleryAsync(client, history, species));
pageBuilder.PictureFilenames.RemoveAll(x => string.IsNullOrWhiteSpace(x));
// Generate page content.
WikiPage wikiPage = await pageBuilder.BuildAsync();
string pageTitle = wikiPage.Title;
bool createRedirect = pageTitle != species.FullName;
// Upload page content.
await _editSpeciesPageAsync(client, history, species, pageTitle, wikiPage.Body);
// Attempt to create the redirect page for the species (if applicable).
if (createRedirect)
{
string redirect_page_title = species.FullName;
if (await _editPageAsync(client, history, redirect_page_title, string.Format("#REDIRECT [[{0}]]", pageTitle) + "\n" + BotFlag))
{
await history.AddRedirectRecordAsync(redirect_page_title, pageTitle);
}
}
_log(string.Format("finished synchronizing species {0}", species.ShortName));
}
}
else
{
_log("mediawiki login failed");
}
_log("synchronizing complete");
await Task.Delay(-1);
}