public async Task <Wikitext> GetArticleMwRootAsync(IWikiArticle wikiArticle)
{
#region Log article
var articleDataString = wikiArticle.ArticleData.Substring(0, Math.Min(wikiArticle.ArticleData.Length, 50));
_logger.LogInformation($"{nameof(GetArticleMwRootAsync)} parsing content:{Environment.NewLine}{articleDataString}");
#endregion
if (wikiArticle.ContentModel != ContentModel.WikiText)
{
var errorMessage = $"This implementation of {nameof(MwParserApi)} can only load an instance of {nameof(IWikiArticle)} with {ContentModel.WikiText} {nameof(ContentModel)}";
_logger.LogError(errorMessage);
throw new NotSupportedException(errorMessage);
}
// Build article AST
_logger.LogDebug("Building article AST...");
var astRoot = await Task.Run(
() => _parser.Parse(wikiArticle.ArticleData));
_logger.LogDebug("Done.");
if (astRoot == null)
{
var errorMessage = $"{nameof(WikitextParser)} returned null AST root.";
_logger.LogError(errorMessage);
throw new ApplicationException(errorMessage);
}
return(astRoot);
}
/// <summary>
/// Parses wikitext, and asserts
/// 1. Whether the parsed AST can be converted back to the same wikitext as input.
/// 2. Whether the parsed AST is correct.
/// </summary>
public Wikitext ParseAndAssert(string text, string expectedDump, WikitextParserOptions options)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
var parser = new WikitextParser {
Options = options
};
var root = parser.Parse(text);
var parsedText = root.ToString();
Output.WriteLine("Original Text\n====================");
Output.WriteLine(text);
Output.WriteLine("Parsed Text\n====================");
Output.WriteLine(parsedText);
var rootExpr = Utility.Dump(root);
Output.WriteLine("AST Dump\n====================");
Output.WriteLine(EscapeString(rootExpr));
if (expectedDump != rootExpr)
{
Assert.Equal(EscapeString(expectedDump), EscapeString(rootExpr));
}
if (!options.AllowClosingMarkInference)
{
Assert.Equal(text, parsedText);
}
return(root);
}
private IEnumerable <WikiPage> GetStoryLinksPageList(WikiSite site, string pageTitle)
{
var targetPage = new WikiPage(site, pageTitle);
targetPage.RefreshAsync(PageQueryOptions.FetchContent | PageQueryOptions.ResolveRedirects).Wait(); //Load page content
//Get page text
var parser = new WikitextParser();
var wikiPageText = parser.Parse(targetPage.Content);
IEnumerable <Template> templateList = new List <Template>();
var header = wikiPageText.Lines.SelectMany(x => x.EnumDescendants().OfType <Heading>()).Where(y => HeadersToSearch.Contains(y.ToPlainText())).SingleOrDefault();
if (header != null)
{
templateList = header.EnumDescendants().OfType <Template>();
}
else
{
templateList = wikiPageText.EnumDescendants().OfType <Template>();
}
var storyLinkTemplates = templateList.Where(template => template.Name.Equals("storylink"));
return(storyLinkTemplates.Select(template => new WikiPage(site, template.Arguments.Single().Value.ToPlainText())));
}
public Wikitext ParseWikitext(string text)
{
var parser = new WikitextParser();
var root = parser.Parse(text);
return(root);
}
/// <summary>
/// Loads a page from file and parse it.
/// </summary>
private static Wikitext LoadAndParse(string fileName)
{
var content = File.ReadAllText(fileName);
var parser = new WikitextParser();
return(parser.Parse(content));
}
public void ParseBadText()
{
var wiki = @"
= Hello =
blah-blah. OK!";
Assert.Throws <Sprache.ParseException>(() => WikitextParser.Parse(wiki));
}
private void UpdatePageContentWithMessage(WikiPage page, string content, string editMessage)
{
var parser = new WikitextParser();
var wikiText = parser.Parse(content);
page.Content = wikiText.ToString();
page.UpdateContentAsync(editMessage).Wait();
}
public static WikiJobRequest GetSampleContinuityLinkFixJobRequest()
{
var parser = new WikitextParser();
var ast = parser.Parse("{{User:Tigerpaw28/Sandbox/Template:WikiBotRequest|type=Link Fix|username=Tigerpaw28|timestamp=14:58, 30 June 2020 (EDT)|before=[[Hot Shot (Armada)]]|after=[[Hot Shot (Armada)/Cartoon continuity]]|headers=Armada cartoon, Energon cartoon, Cybertron cartoon|media=Cartoon|pages=Optimus Prime (Armada)|comment=Test job|status=PendingPreApproval}}");
var templates = ast.Lines.First <LineNode>().EnumDescendants().OfType <Template>();
var request = WikiJobRequestFactory.GetWikiJobRequest(JobType.ContinuityLinkFixJob, TimeZoneInfo.Local, templates.First());
return(request);
}
public static WikiJobRequest GetSampleLinkFixJobRequest()
{
var parser = new WikitextParser();
var ast = parser.Parse("{{User:Tigerpaw28/Sandbox/Template:WikiBotRequest|type=Link Fix|username=Tigerpaw28|timestamp=14:58, 30 June 2020 (EDT)|before=[[Commercial]]|after=[[Commercial|Test]]|pages=Commercial|comment=Test job|status=PendingPreApproval}}");
var templates = ast.Lines.First <LineNode>().EnumDescendants().OfType <Template>();
var request = WikiJobRequestFactory.GetWikiJobRequest(JobType.LinkFixJob, TimeZoneInfo.Local, templates.First());
return(request);
}
public void ParseSimpleTextTest()
{
var wiki = "blah-blah. OK!";
var doc = WikitextParser.Parse(wiki);
var elements = doc.Elements.Cast <WikitextPlainText>().ToArray();
Assert.Single(elements);
Assert.Equal(wiki, elements[0].Value);
}
public async Task <Entity> QueryByNameAsync(string name)
{
if (name == null)
{
throw new ArgumentNullException(nameof(name));
}
name = name.Trim();
if (name == "")
{
return(null);
}
var site = await WikiFamily.GetSiteAsync(LanguageCode);
var page = await FetchPageAsync(site, name);
if (page == null)
{
return(null);
}
var parser = new WikitextParser();
var root = parser.Parse(page.Content);
var template = root.EnumDescendants().OfType <Template>()
.FirstOrDefault(t => distinguishingTemplates.Contains(Utility.NormalizeTitle(t.Name)));
Entity entity;
if (template != null)
{
switch (Utility.NormalizeTitle(template.Name))
{
case "Book":
entity = BuildVolume(root);
break;
case "Charcat":
entity = BuildCat(root);
break;
default:
Debug.Assert(false);
return(null);
}
}
else if (await page.IsDisambiguationAsync())
{
entity = BuildDisambiguation(root);
}
else
{
entity = BuildUnknown(root);
}
entity.Name = page.Title;
entity.DetailUrl = Utility.GetPageUrl(site, page.Title);
return(entity);
}
private static void ParseAndPrint()
{
var parser = new WikitextParser();
Console.WriteLine("Please input the wikitext to parse; use EOF (Ctrl+Z) to accept:");
var ast = parser.Parse(ReadInput());
Console.WriteLine("Parsed AST");
PrintAst(ast, 0);
Console.WriteLine("Plain text");
Console.WriteLine(ast.ToPlainText());
}
private WikiJobRequest BuildRequest()
{
var parser = new WikitextParser();
var ast = parser.Parse("{{User:Tigerpaw28/Sandbox/Template:WikiBotRequest|type=Text Replacement|username=Tigerpaw28|timestamp=14:58, 30 June 2020 (EDT)|before=Deceptitran|after=not a Robot|comment=Test job|status=PendingPreApproval}}");
var templates = ast.Lines.First <LineNode>().EnumDescendants().OfType <Template>();
var request = WikiJobRequestFactory.GetWikiJobRequest(JobType.TextReplacementJob, TimeZoneInfo.Local, templates.First());
List <Page> pages = new List <Page>();
pages.Add(new Page(0, "Test"));
pages.Add(new Page(1, "Commercial"));
request.Pages = pages;
return(request);
}
public void ParseTextWithHeader()
{
var wiki = @"
= Hello =
blah-blah. OK!";
var doc = WikitextParser.Parse(wiki);
var elements = doc.Elements.Cast <IWikitextValue>().ToArray();
Assert.Equal(2, elements.Length);
Assert.Equal("Hello", elements[0].Value);
Assert.Equal("blah-blah. OK!", elements[1].Value);
}
static async Task WorkAsync(IAsyncEnumerable <WikiPage> pages)
{
const string templateName = "WbClientLite/SiteLinks";
var parser = new WikitextParser();
var counter = 0;
using (var ie = pages.GetEnumerator())
{
while (await ie.MoveNext())
{
var page = ie.Current;
counter++;
Console.Write("{0}: {1} ", counter, page);
var root = parser.Parse(page.Content);
if (root.EnumDescendants().OfType <Template>().Any(t => MwParserUtility.NormalizeTitle(t.Name) == templateName))
{
Console.WriteLine("Skipped");
continue;
}
var langLinks = root.EnumDescendants().OfType <WikiLink>().Where(l =>
{
var wl = WikiClientLibrary.WikiLink.Parse(page.Site, l.Target.ToString());
return(wl.InterwikiPrefix != null);
}).ToList();
// Remove old language links.
foreach (var link in langLinks)
{
if (link.PreviousNode is PlainText pt1 && string.IsNullOrWhiteSpace(pt1.Content))
{
pt1.Remove();
}
if (link.NextNode is PlainText pt2 && string.IsNullOrWhiteSpace(pt2.Content))
{
pt2.Remove();
}
var parent = link.ParentNode;
link.Remove();
if (!parent.EnumChildren().Any())
{
parent.Remove();
}
}
// Insert new template.
root.Lines.Add(new Paragraph(new PlainText("\n"), new Template(new Run(new PlainText(templateName)))));
page.Content = root.ToString();
await page.UpdateContentAsync("使用CrystalPool提供的语言链接。", true, true);
Console.WriteLine("Done");
}
}
}
static void SimpleDemo()
{
// Fills the missing template parameters.
var parser = new WikitextParser();
var templateNames = new [] { "Expand section", "Cleanup" };
var text = @"==Hello==<!--comment-->
{{Expand section|
date=2010-10-05
}}
{{Cleanup}}
This is a nice '''paragraph'''.
==References==
{{Reflist}}
";
var ast = parser.Parse(text);
// Convert the code snippets to nodes
var dateName = parser.Parse("date");
var dateValue = parser.Parse(DateTime.Now.ToString("yyyy-MM-dd"));
Console.WriteLine("Issues:");
// Search and set
foreach (var t in ast.EnumDescendants().OfType <Template>()
.Where(t => templateNames.Contains(MwParserUtility.NormalizeTemplateArgumentName(t.Name))))
{
// Get the argument by name.
var date = t.Arguments["date"];
if (date != null)
{
// To print the wikitext instead of user-friendly text, use ToString()
Console.WriteLine("{0} ({1})", t.Name.ToPlainText(), date.Value.ToPlainText());
}
// Update/Add the argument
t.Arguments.SetValue(dateName, dateValue);
}
Console.WriteLine();
Console.WriteLine("Wikitext:");
Console.WriteLine(ast.ToString());
}
protected override void OnRefreshDocumentOutline()
{
base.OnRefreshDocumentOutline();
// Show document headings
var parser = new WikitextParser();
var documentText = TextBox.Text;
Heading[] headings = null;
if (!string.IsNullOrWhiteSpace(documentText))
{
var root = parser.Parse(documentText);
headings = root.EnumDescendants().OfType <Heading>().ToArray();
}
Dispatcher.AutoInvoke(() =>
{
DocumentOutline.Clear();
if (headings == null)
{
return;
}
var levelStack = new Stack <Tuple <Heading, DocumentOutlineItem> >();
foreach (var h in headings)
{
var outline = new DocumentOutlineItem
{
Text = string.Join(null, h.Inlines).Trim(),
OutlineContext = h
};
outline.DoubleClick += OutlineItem_DoubleClick;
while (levelStack.Count > 0)
{
var lastLevel = levelStack.Peek().Item1.Level;
if (lastLevel < h.Level)
{
// Append as child item.
levelStack.Peek().Item2.Children.Add(outline);
goto NEXT;
}
// Sibling or upper levels.
levelStack.Pop();
}
// levelStack.Count == 0
DocumentOutline.Add(outline);
levelStack.Push(Tuple.Create(h, outline));
NEXT:
;
}
});
}
private static void ParseAndPrint(List <Page> pages, string title)
{
var page = pages.FirstOrDefault(x => x.Title == title);
if (page == null)
{
Console.WriteLine("Page '{0}' not found.", title);
return;
}
var parser = new WikitextParser();
var ast = parser.Parse(page.Text);
// PrintAst(ast, 0);
Console.WriteLine(ast.ToPlainText());
}
/// <summary>
/// Builds <see cref="PageInfo"/> from a MediaWiki page with its content fetched.
/// </summary>
public static PageInfo BuildBasicInfo(Page page, WikitextParser parser)
{
if (page == null)
{
throw new ArgumentNullException(nameof(page));
}
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
var info = new PageInfo
{
Title = page.Title,
LastRevisionId = page.LastRevisionId,
LastRevisionTime = page.LastRevision?.TimeStamp ?? DateTime.MinValue,
LastRevisionUser = page.LastRevision?.UserName,
ContentLength = page.ContentLength,
};
if (page.IsRedirect)
{
info.Description = page.Content;
}
else if (!string.IsNullOrWhiteSpace(page.Content))
{
if (page.InferContentModel() == ContentModels.Wikitext)
{
var p = parser.Parse(page.Content);
// Search for leading line.
var leadingLine = p.EnumChildren()
.OfType <Paragraph>()
.FirstOrDefault(line => line.Inlines.OfType <PlainText>()
.Any(pt => !string.IsNullOrWhiteSpace(pt.Content)));
if (leadingLine != null)
{
info.Description = leadingLine.ToString();
}
// Collect template arguments, if any.
info.TemplateArguments = p.EnumDescendants()
.OfType <ArgumentReference>()
.Select(r => new TemplateArgumentInfo(r.Name.ToString().Trim()))
.Distinct(TemplateArgumentInfoComparer.Default)
.ToArray();
}
}
return(info);
}
private void Parse()
{
if (TextView != null)
{
var parser = new WikitextParser();
var text = TextView.Dispatcher.AutoInvoke(() => TextView.Document.Text);
var sw = Stopwatch.StartNew();
var ast = parser.Parse(text);
Trace.WriteLine("Parsed " + text.Length + " chars in " + sw.Elapsed);
documentAstInvalidated = false;
TextView.Dispatcher.BeginInvoke((Action)(() =>
{
AstRoot = ast;
TextView.Redraw();
}));
}
documentAstInvalidated = false;
}
public static async Task <string> GetWikiPage(string title)
{
var requestContent = new Dictionary <string, string>
{
{ "format", "json" },
{ "action", "query" },
{ "prop", "revisions" },
{ "rvlimit", "1" },
{ "rvprop", "content" },
{ "titles", title },
{ "redirects", "resolve" }
};
using (var client = new HttpClient())
using (var postVars = new FormUrlEncodedContent(requestContent))
{
var response = await client.PostAsync(endpointUrl, postVars);
var stringContent = await response.Content.ReadAsStringAsync();
var root = JObject.Parse(stringContent);
var content = (string)root["query"]["pages"].Children <JProperty>().First().Value["revisions"][0]["*"];
var parser = new WikitextParser();
var result = parser.Parse(content);
var lines = result.EnumChildren().ToList();
var textLines = lines.Select(c => ExtractStringFromQuery(c))
.Where(c => c != "\n")
.Select(c => c.Trim())
.ToList();
var taken = textLines
.SkipWhile(c => String.IsNullOrEmpty(c))
.TakeWhile(c => !String.IsNullOrEmpty(c) && !c.StartsWith("**"))
.Take(3)
.ToList();
return(string.Join("\n", taken));
}
}
public static List <WikiJobRequest> GetSampleJobRequests(bool includePages)
{
var requests = new List <WikiJobRequest>();
var array = GetRawRequestArray();
var parser = new WikitextParser();
for (int x = 0; x < 5; x++)
{
var ast = parser.Parse(array[x]);
var templates = ast.Lines.First <LineNode>().EnumDescendants().OfType <Template>();
var request = WikiJobRequestFactory.GetWikiJobRequest(JobType.TextReplacementJob, TimeZoneInfo.Local, templates.First());
request.ID = x + 2;
if (!includePages)
{
request.Pages = null;
}
requests.Add(request);
}
return(requests);
}
/// <summary>
/// Fetches a page from en Wikipedia, and parse it.
/// </summary>
private static Wikitext FetchAndParse(string title)
{
if (title == null)
{
throw new ArgumentNullException(nameof(title));
}
const string EndPointUrl = "https://en.wikipedia.org/w/api.php";
var client = new HttpClient();
var requestContent = new Dictionary <string, string>
{
{ "format", "json" },
{ "action", "query" },
{ "prop", "revisions" },
{ "rvlimit", "1" },
{ "rvprop", "content" },
{ "titles", title }
};
var response = client.PostAsync(EndPointUrl, new FormUrlEncodedContent(requestContent)).Result;
var root = JObject.Parse(response.Content.ReadAsStringAsync().Result);
var content = (string)root["query"]["pages"].Children <JProperty>().First().Value["revisions"][0]["*"];
var parser = new WikitextParser();
return(parser.Parse(content));
}
public IEnumerable <string> FactsFromPage(WikiPage page)
{
var root = wikiParser.Parse(page.Content);
var infoboxCat = root.EnumDescendants().OfType <Template>()
.FirstOrDefault(t => MwParserUtility.NormalizeTitle(t.Name) == "Charcat");
if (infoboxCat == null)
{
Console.WriteLine("No {{Charcat}} found.");
yield return("% No {{Charcat}} found.");
yield break;
}
var atom = AtomExpr(page.Title.Trim());
yield return($"name({atom}, \"{StripDabTitle(page.Title)}\").");
switch (EnWikiHelper.IsTom(root))
{
case true:
yield return($"male({atom}).");
break;
case false:
yield return($"female({atom}).");
break;
default:
break;
}
var affie = infoboxCat.Arguments["affie"];
if (affie != null)
{
foreach (var aff in affie.EnumDescendants().OfType <WikiLink>())
{
yield return($"belongsto({atom}, {AtomExpr(aff.Target.ToPlainText().Trim())}).");
}
}
var familyt = infoboxCat.Arguments["familyt"];
var familyl = infoboxCat.Arguments["familyl"];
if (familyt != null && familyl != null)
{
var familyDict = EnWikiHelper.ParseFamily(familyt.Value, familyl.Value);
familyDict.TryGetValue("mother", out var mothers);
familyDict.TryGetValue("father", out var fathers);
var mother = mothers?.FirstOrDefault();
var father = fathers?.FirstOrDefault();
if (mother != null || father != null)
{
Debug.Assert(mothers == null || mothers.Count <= 1);
Debug.Assert(fathers == null || fathers.Count <= 1);
yield return($"child({atom}, {AtomExpr(father ?? "x")}, {AtomExpr(mother ?? "x")}).");
}
else
{
yield return("% No parent found.");
}
}
else
{
Console.WriteLine("No familyt/familyl.");
yield return("% No {{Charcat |familyl= |familyt= }} found.");
}
var apps = infoboxCat.Arguments["mentor"];
if (apps != null)
{
foreach (var appr in apps.EnumDescendants().OfType <WikiLink>())
{
yield return($"apprentice({atom}, {AtomExpr(appr.Target.ToPlainText().Trim())}).");
}
}
}
public async void UpdateRequests(List <WikiJobRequest> requests)
{
using (var client = new WikiClient
{
ClientUserAgent = "WCLQuickStart/1.0 (your user name or contact information here)"
})
{
try
{
// You can create multiple WikiSite instances on the same WikiClient to share the state.
var site = _wikiAccessLogic.GetLoggedInWikiSite(_wikiLoginConfig, client, _log);
var page = new WikiPage(site, _wikiRequestPage);
_log.Information("Pulling requests from job request page for status update.");
// Fetch content of job request page so we can update it
await page.RefreshAsync(PageQueryOptions.FetchContent
| PageQueryOptions.ResolveRedirects);
var parser = new WikitextParser();
var wikiText = parser.Parse(page.Content);
foreach (WikiJobRequest request in requests)
{
_log.Information($"Processing request ID: {request.ID} with raw {request.RawRequest}");
//Find corresponding template in the page content
var templates = wikiText.Lines.SelectMany(x => x.EnumDescendants().OfType <Template>());
var requestTemplates = templates.Where(template => template.Name.ToPlainText().Equals(_botRequestTemplate));
_log.Information($"{requestTemplates.ToList().Count} templates found for template {_botRequestTemplate}");
_log.Information($"Template id: {requestTemplates.First().Arguments.SingleOrDefault(arg => arg.Name.ToPlainText().Equals("id"))}");
var singletemplate = requestTemplates.First(template => template.EqualsJob(request));
if (singletemplate.Arguments.SingleOrDefault(arg => arg.Name.ToPlainText().Equals("status")) == null) //Status argument doesn't exist in the template
{
var templateArgument = new TemplateArgument {
Name = parser.Parse("status"), Value = parser.Parse(request.Status.ToString())
};
singletemplate.Arguments.Add(templateArgument);
}
else //Status argument exists
{
singletemplate.Arguments.Single(arg => arg.Name.ToPlainText().Equals("status")).Value = parser.Parse(request.Status.ToString());
}
if (singletemplate.Arguments.SingleOrDefault(arg => arg.Name.ToPlainText().Equals("id")) == null) //ID argument doesn't exist in the template
{
var templateArgument = new TemplateArgument {
Name = parser.Parse("id"), Value = parser.Parse(request.ID.ToString())
};
singletemplate.Arguments.Add(templateArgument);
}
request.RawRequest = singletemplate.ToString();
_database.UpdateRaw(request.ID, request.RawRequest); //TODO: Make batch operation
}
//Update the content of the page object and push it live
await UpdatePageContent(wikiText.ToString(), "Updating request ids and statuses", page);
// We're done here
await site.LogoutAsync();
}
catch (Exception ex)
{
_log.Error(ex, "An error occurred while trying to update requests: ");
}
}
}
public async Task PopulateRelationsAsync()
{
var processedEntities = GetProcessedEntities();
await zhWarriorsSite.Initialization;
var counter = 0;
foreach (var catg in GetCatsToProcess(processedEntities).Buffer(50))
{
await catg.Select(t => t.ZhPage).RefreshAsync(PageQueryOptions.FetchContent);
foreach (var(id, title, page) in catg)
{
counter++;
Logger.LogInformation("[{}] Processing {} -> {}", counter, title, id);
try
{
await EditEntityAsync(new Entity(Site, id), page);
processedEntities.Add(id);
}
catch (KeyNotFoundException)
{
Logger.LogWarning("Missing entity.");
}
WriteProcessedEntities(processedEntities);
}
}
async Task EditEntityAsync(Entity entity, WikiPage page)
{
var root = parser.Parse(page.Content);
var infobox = root.EnumDescendants().TemplatesWithTitle("Infobox cat").FirstOrDefault();
if (infobox == null)
{
Logger.LogError("No {{Infobox cat}} found.");
return;
}
var father = infobox.Arguments["father"]?.Value.EnumDescendants().OfType <WikiLink>().FirstOrDefault()?.Target.ToPlainText();
var mother = infobox.Arguments["mother"]?.Value.EnumDescendants().OfType <WikiLink>().FirstOrDefault()?.Target.ToPlainText();
var mates = infobox.Arguments["mate"]?.Value.EnumDescendants().OfType <WikiLink>().Select(l => l.Target.ToPlainText()).ToList();
var fosters = infobox.Arguments["foster_father"]?.Value.EnumDescendants()
.Concat(infobox.Arguments["foster_mother"]?.Value.EnumDescendants() ?? Enumerable.Empty <Node>())
.OfType <WikiLink>().Select(l => l.Target.ToPlainText()).ToList();
var mentors = infobox.Arguments["mentor"]?.Value.EnumDescendants().OfType <WikiLink>().Select(l => l.Target.ToPlainText()).ToList();
Console.WriteLine(father);
Console.WriteLine(mother);
Console.WriteLine(string.Join(";", mates));
Console.WriteLine(string.Join(";", fosters));
Console.WriteLine(string.Join(";", mentors));
var claims = new List <Claim>();
if (father != null)
{
var f = CPRepository.EntityFromZhSiteLink(father);
if (f == null)
{
WriteMissingEntity(father);
throw new KeyNotFoundException();
}
claims.Add(new Claim("P88", f, BuiltInDataTypes.WikibaseItem));
}
if (mother != null)
{
var m = CPRepository.EntityFromZhSiteLink(mother);
if (m == null)
{
WriteMissingEntity(mother);
throw new KeyNotFoundException();
}
claims.Add(new Claim("P89", m, BuiltInDataTypes.WikibaseItem));
}
if (fosters != null)
{
foreach (var foster in fosters)
{
var f = CPRepository.EntityFromZhSiteLink(foster);
if (f == null)
{
WriteMissingEntity(foster);
throw new KeyNotFoundException();
}
claims.Add(new Claim("P99", f, BuiltInDataTypes.WikibaseItem));
}
}
if (mates != null)
{
var index = 1;
foreach (var mate in mates)
{
var f = CPRepository.EntityFromZhSiteLink(mate);
if (f == null)
{
WriteMissingEntity(mate);
throw new KeyNotFoundException();
}
claims.Add(new Claim("P100", f, BuiltInDataTypes.WikibaseItem)
{
Qualifiers = { new Snak("P53", index.ToString(), BuiltInDataTypes.String) }
});
index++;
}
}
if (mentors != null)
{
foreach (var mentor in mentors)
{
var f = CPRepository.EntityFromZhSiteLink(mentor);
if (f == null)
{
WriteMissingEntity(mentor);
throw new KeyNotFoundException();
}
claims.Add(new Claim("P86", f, BuiltInDataTypes.WikibaseItem));
}
}
if (claims.Any())
{
await entity.EditAsync(claims.Select(c => new EntityEditEntry(nameof(entity.Claims), c)),
"Populate relations from zhwarriorswiki.", EntityEditOptions.Bot);
}
}
}
public override void Execute()
{
SetJobStart();
try
{
using (var client = new WikiClient())
{
var site = _wikiAccessLogic.GetLoggedInWikiSite(WikiConfig, client, Log);
var parser = new WikitextParser();
var wikiText = parser.Parse(FromText);
var fromLinkTarget = wikiText.Lines.SelectMany(x => x.EnumDescendants().OfType <WikiLink>()).FirstOrDefault().Target.ToPlainText();
var PageList = GetBackLinksPageList(site, fromLinkTarget);
string filename = "";
string diff = "";
string filePath = "";
var folderName = Request.ID.ToString();
var folderPath = Path.Combine(Configuration["DiffDirectory"], folderName);
if (!Directory.Exists(folderPath))
{
Directory.CreateDirectory(folderPath);
}
foreach (WikiPage page in PageList)
{
Log.Information("Processing page {PageName}", page.Title);
page.RefreshAsync(PageQueryOptions.FetchContent | PageQueryOptions.ResolveRedirects).Wait(); //Load page content
var beforeContent = page.Content;
var wikiPageText = parser.Parse(beforeContent);
IEnumerable <WikiLink> wikiLinks = null;
if (string.IsNullOrWhiteSpace(string.Join(' ', HeadersToSearch)))
{
wikiLinks = wikiPageText.Lines.SelectMany(x => x.EnumDescendants().OfType <WikiLink>());
}
else
{
var header = wikiPageText.Lines.SelectMany(x => x.EnumDescendants().OfType <Heading>()).Where(y => y.ToPlainText().Equals(HeadersToSearch)).Single();
wikiLinks = header.EnumDescendants().OfType <WikiLink>();
}
var matchingLinks = wikiLinks.Where(link => CompareLinks(link.Target.ToString(), fromLinkTarget)).ToList();
if (!matchingLinks.Any() || page.Title.Equals(Configuration["WikiRequestPage"], StringComparison.OrdinalIgnoreCase))
{
Request.Pages.RemoveAll(x => x.Name.Equals(page.Title, StringComparison.OrdinalIgnoreCase));
}
else
{
foreach (WikiLink link in matchingLinks)
{
Log.Debug($"Link target starts: {link.Target}");
var newTarget = parser.Parse(ToText).Lines.SelectMany(x => x.EnumDescendants().OfType <WikiLink>()).FirstOrDefault().Target.ToPlainText();
if (link.Text == null && (!link.Target.ToPlainText().Contains("(") && newTarget.Contains("(")))
{
link.Text = new Run(new PlainText(link.Target.ToPlainText())); //Maintain original link text if the link had no custom text and no disambig
}
link.Target = new Run(new PlainText(newTarget));
Log.Debug($"Link target ends: {link.Target}");
}
Log.Debug($"Content after: {wikiPageText}");
var afterContent = wikiPageText.ToString();
if (Request.Status != JobStatus.Approved) //Create diffs for approval
{
Log.Information("Generating diff for page {PageName}", page.Title);
Utilities.GenerateAndSaveDiff(beforeContent, afterContent, page.Title, Request.ID, Configuration["DiffDirectory"], folderName);
//var wikiDiff = new WikiDiff();
//diff = $"{WikiDiff.DiffHead()}</head><body>{WikiDiff.TableHeader}{wikiDiff.GetDiff(beforeContent, afterContent, 1)}</table></body></html>";
//filename = "Diff-" + Request.ID + "-" + page.Title + ".txt"; //Set filename for this page
//filename = Utilities.SanitizeFilename(filename, '_');
//filePath = Path.Combine(Configuration["DiffDirectory"], folderName, filename);
//File.WriteAllText(filePath, diff);
JobData.SaveWikiJobRequest(Request); //Save page list
}
else //Apply changes
{
Log.Information("Applying replacement for page {PageName}", page.Title);
var editMessage = $"{WikiConfig["Username"]} Text Replacement {FromText} => {ToText}";
((TFWikiJobRetriever)Retriever).UpdatePageContent(afterContent, editMessage, page).Wait();
}
}
Thread.Sleep(1000 * _throttleSpeedInSeconds);
}
}
Thread.Sleep(1000);
}
catch (Exception ex)
{
Request.Status = JobStatus.Failed;
Log.Error(ex, $"TextReplacementJob with ID: {Request.ID} failed.");
}
finally
{
SetJobEnd();
SaveRequest();
}
}
public override void Execute()
{
SetJobStart();
List <WikiPage> AlreadyUpdatedPages = new List <WikiPage>();
List <WikiPage> NoUpdateNeededPages = new List <WikiPage>();
try
{
using (var client = new WikiClient())
{
var site = _wikiAccessLogic.GetLoggedInWikiSite(WikiConfig, client, Log);
var parser = new WikitextParser();
var wikiText = parser.Parse(FromText);
var fromLinkTarget = wikiText.Lines.SelectMany(x => x.EnumDescendants().OfType <WikiLink>()).FirstOrDefault().Target.ToPlainText();
wikiText = parser.Parse(ToText);
var toLinkTarget = wikiText.Lines.SelectMany(x => x.EnumDescendants().OfType <WikiLink>()).FirstOrDefault().Target.ToPlainText();
var PageList = GetBackLinksPageList(site, fromLinkTarget);
var storyLinkPageList = GetStoryLinksPageList(site, toLinkTarget);
//string filename = "";
//string diff = "";
//string filePath = "";
var folderName = Request.ID.ToString();
var folderPath = Path.Combine(Configuration["DiffDirectory"], folderName);
if (!Directory.Exists(folderPath))
{
Directory.CreateDirectory(folderPath);
}
foreach (WikiPage page in PageList)
{
Log.Information("Processing page {PageName}", page.Title);
IEnumerable <WikiLink> wikiLinks = null;
page.RefreshAsync(PageQueryOptions.FetchContent | PageQueryOptions.ResolveRedirects).Wait(); //Load page content
//Get page text
var beforeContent = page.Content;
var wikiPageText = parser.Parse(beforeContent);
if (string.IsNullOrWhiteSpace(string.Join(' ', HeadersToSearch)))
{
throw new Exception("No continuity header specified");
}
else
{
//Get any wiki links under the header
var headers = wikiPageText.EnumDescendants().OfType <Heading>();
var matchingHeaders = headers.Where(y => HeadersToSearch.Contains(y.ToPlainText()) || HeadersToSearch.Contains(y.ToString()));
//Need to handle cases like Armada Megatron where links are on cartoon/comic/whatever subpage. Look for subpage header based on media type, get subpage, add to Pages list and get links.
if (matchingHeaders.Any())
{
var contentNodes = GetContentBetweenHeaders(headers, matchingHeaders, wikiPageText);
wikiLinks = contentNodes.SelectMany(x => x.EnumDescendants().OfType <WikiLink>());
}
else
{
var templates = wikiPageText.Lines.SelectMany(x => x.EnumDescendants().OfType <Template>());
var mediaTemplate = templates.Where(template => template.Name.ToPlainText().Equals(GetTemplateNameByMedia(Media))).SingleOrDefault();
if (mediaTemplate != null || HasFileTemplateForMedia(templates, Media))
{
wikiLinks = wikiPageText.Lines.SelectMany(x => x.EnumDescendants().OfType <WikiLink>());
}
else
{
var mainTemplates = templates.Where(template => template.Name.ToPlainText().Equals("main"));
if (mainTemplates.Any())
{
wikiLinks = GetMainTemplatePageLinks(mainTemplates, site);
}
}
}
}
var matchingLinks = wikiLinks?.Where(link => CompareLinks(link.Target.ToString(), fromLinkTarget)).ToList();
if (matchingLinks == null || !matchingLinks.Any() || page.Title.Equals(Configuration["WikiRequestPage"], StringComparison.OrdinalIgnoreCase))
{
Request.Pages.RemoveAll(x => x.Name.Equals(page.Title, StringComparison.OrdinalIgnoreCase));
if (wikiLinks?.Where(link => CompareLinks(link.Target.ToString(), toLinkTarget)).Any() ?? false)
{
AlreadyUpdatedPages.Add(page);
}
else
{
NoUpdateNeededPages.Add(page);
}
}
else
{
foreach (WikiLink link in matchingLinks)
{
Log.Debug($"Link target starts: {link.Target}");
var newTarget = parser.Parse(ToText).Lines.SelectMany(x => x.EnumDescendants().OfType <WikiLink>()).FirstOrDefault().Target.ToPlainText();
if (link.Text == null)
{
link.Text = new Run(new PlainText(link.Target.ToPlainText())); //Maintain original link text if the link had no custom text
}
link.Target = new Run(new PlainText(newTarget));
Log.Debug($"Link target ends: {link.Target}");
}
Log.Debug($"Content after: {wikiPageText}");
var afterContent = wikiPageText.ToString();
if (Request.Status != JobStatus.Approved) //Create diffs for approval
{
Log.Information("Generating diff for page {PageName}", page.Title);
Utilities.GenerateAndSaveDiff(beforeContent, afterContent, page.Title, Request.ID, Configuration["DiffDirectory"], folderName);
JobData.SaveWikiJobRequest(Request); //Save page list
}
else //Apply changes
{
Log.Information("Applying replacement for page {PageName}", page.Title);
var editMessage = $"{WikiConfig["Username"]} Text Replacement {FromText} => {ToText}";
((TFWikiJobRetriever)Retriever).UpdatePageContent(afterContent, editMessage, page).Wait();
}
}
Thread.Sleep(1000 * _throttleSpeedInSeconds);
}
}
Thread.Sleep(1000);
}
catch (Exception ex)
{
Request.Status = JobStatus.Failed;
Log.Error(ex, $"TextReplacementJob with ID: {Request.ID} failed.");
}
finally
{
SetJobEnd();
SaveRequest();
}
}