public void Execute(IEnumerable<string> args)
{
var wikiParser = new WikiParser();
City city;
switch (_city)
{
case Constants.CityIds.Moscow:
city = City.Moscow();
city.MetroLines = wikiParser.GetMoscowMetroSchema();
break;
case Constants.CityIds.SaintPetersburg:
city = City.SaintPetersburg();
city.MetroLines = wikiParser.GetSpbMetroSchema();
break;
default:
throw new CityNotFound(_city);
}
var xmlSerializer = new XmlSerializer(typeof(City));
var settings = new XmlWriterSettings
{
Indent = true,
IndentChars = " ",
NewLineChars = "\r\n",
NewLineHandling = NewLineHandling.Replace
};
using (XmlWriter writer = XmlWriter.Create(_filepath, settings))
{
xmlSerializer.Serialize(writer, city);
}
}
public ContoursElevationRuleMap Parse(Stream inputStream)
{
WikiParser.SetWikiContentSource(inputStream);
rules = new ContoursElevationRuleMap();
templateCounter = 0;
string currentMainSection = null;
while (true)
{
WikiContentType wikiContentType;
wikiContentType = WikiParser.Next();
if (wikiContentType == WikiContentType.Eof)
{
break;
}
if (wikiContentType == WikiContentType.Section || wikiContentType == WikiContentType.Text)
{
WikiSection section = WikiParser.Context.LowestSection;
if (section == null)
{
continue;
}
if (wikiContentType == WikiContentType.Section)
{
if (section.SectionLevel == 1)
{
currentMainSection = section.SectionName;
}
}
else if (wikiContentType == WikiContentType.Text)
{
continue;
}
if (currentMainSection == null)
{
continue;
}
switch (currentMainSection.ToLowerInvariant())
{
case "contours rendering rules":
{
SkipToTableAndParseIt(ParseRule);
break;
}
}
}
}
return(rules);
}
public static Dictionary <string, Player> GetPlayerInfoFromParticipants(ICarnoServiceSink sink, string s)
{
List <WikiNode> nodes = WikiParser.Parse(s).ToList();
var participants = new Dictionary <string, Player>();
bool taking = false;
foreach (var node in nodes)
{
if (node.IsSection())
{
taking = ((node as WikiTextNode).Text == "Participants");
}
if (taking && node is WikiTableNode)
{
foreach (var cell in (node as WikiTableNode).Cells)
{
var cellnodes = (from cellnode in cell
where cellnode is WikiTemplateNode
let nodetemplate = cellnode as WikiTemplateNode
orderby nodetemplate.Name
select nodetemplate).ToList();
if (cellnodes.Count == 0 || cellnodes.Count > 2 ||
cellnodes[0].Name.ToLower() != "player")
{
continue;
}
WikiTemplateNode player = cellnodes[0];
string playerId = GetPlayerId(player);
string team = "noteam";
if (cellnodes.Count == 2)
{
WikiTemplateNode temppart = cellnodes[1];
team = temppart.Name.From("TeamPart/");
team = sink.ConformTeamId(team);
}
Player info = new Player();
info.Id = player.GetParamText("1");
info.Flag = player.GetParamText("flag");
info.Team = team;
info.Link = LiquipediaUtils.NormaliseLink(GetPlayerLink(player.GetParamText("link")));
participants[playerId] = info;
if (!string.IsNullOrEmpty(info.Link))
{
sink.SetIdLinkMap(info.Id, info.Link);
}
}
}
}
return(participants);
}
/// <summary>
/// Initializes a new instance of the <see cref="WikiRulesParser"/> class.
/// </summary>
/// <param name="inputStream">The input stream from which the rendering rules will be parsed..</param>
public WikiRulesParser(
Stream inputStream,
TypesRegistry typesRegistry,
CharactersConversionDictionary charactersConversionDictionary,
ISerializersRegistry serializersRegistry)
{
this.typesRegistry = typesRegistry;
this.charactersConversionDictionary = charactersConversionDictionary;
this.serializersRegistry = serializersRegistry;
WikiParser.SetWikiContentSource(inputStream);
}
public static MovieInfo GetMovieInfo(string movieName, string outputDirectory)
{
MovieInfo info = new MovieInfo()
{
Name = movieName
};
ChromeOptions options = new ChromeOptions();
//options.AddArgument("headless");
using (ChromeDriver chrome = new ChromeDriver(Environment.CurrentDirectory, options))
{
//Search Movie
Searcher searcher = new Searcher(driver: chrome,
urlBuilder: SearchUrlBuilders.Bing,
parsers: (driver) => RankWikiLinks(driver, movieName));
var result = searcher.Search($"{movieName} film").Select(link => link.GetAttribute("href")).ToList();
foreach (var link in result)
{
try
{
info.WikiLink = link;
chrome.Navigate().GoToUrl(info.WikiLink);
chrome.WaitForPageLoad();
//Extract Info
info.WikiScreenShotPath = Path.Combine(outputDirectory, $"{movieName}_wiki.png");
chrome.GetScreenshot().SaveAsFile(info.WikiScreenShotPath);
WikiParser wp = new WikiParser(chrome);
info.Directors_Wiki = wp.GetDirectors();
//Extract Info
var imdbLink = LinkFinders.FindImdb(chrome)?.Reverse()?.First();
info.ImdbLink = imdbLink.GetAttribute("href");
chrome.Navigate().GoToUrl($"{info.ImdbLink}fullcredits");
info.ImdbScreenShotPath = Path.Combine(outputDirectory, $"{movieName}_imdb.png");
chrome.GetScreenshot().SaveAsFile(info.ImdbScreenShotPath);
ImdbParser ip = new ImdbParser(chrome);
info.Directors_Imdb = ip.GetDirectors();
break;
}
catch (Exception e)
{
Console.WriteLine($"{movieName}" + e.Message);
}
}
}
return(info);
}
protected DataSet GetPageText(bool FormatWikiText)
{
using (
SqlDataAdapter da = new SqlDataAdapter(
"w_GetPage",
ConfigurationManager.ConnectionStrings["ConnectionString"].ConnectionString
))
{
da.SelectCommand.CommandType = CommandType.StoredProcedure;
da.SelectCommand.Parameters.Add("@PageName", SqlDbType.VarChar).Value = pageName;
da.SelectCommand.Parameters.Add("@IncrementHitCount", SqlDbType.Bit).Value = Page.IsPostBack ? 0 : 1;
DataSet ds = new DataSet();
da.Fill(ds, "Wiki");
if (ds.Tables["Wiki"] != null && ds.Tables["Wiki"].Rows.Count > 0)
{
try
{
//Get the links from the table
da.SelectCommand.CommandText = "w_GetLinks";
da.SelectCommand.Parameters.Clear();
da.Fill(ds, "Links");
//Add the wiki formatted text to the dataset
ds.Tables["Wiki"].Columns.Add("PageView");
wikiStartTime = System.DateTime.Now;
if (FormatWikiText)
{
ds.Tables["Wiki"].Rows[0]["PageView"] = WikiParser.GenerateWikiText(ds.Tables["Wiki"].Rows[0]["PageText"].ToString(), ds.Tables["Links"]);
}
wikiEndTime = System.DateTime.Now;
if (ConfigurationManager.AppSettings["displayPageStats"].ToString().ToLower() == "true")
{
lblPageStats.Text = string.Format("Wiki formatted text generation: {0}", wikiEndTime - wikiStartTime);
}
return(ds);
}
catch (Exception ex)
{
HandleError(ex);
return(null);
}
}
else
{
return(null);
}
}
}
protected void btnPreviewInsert_Click(object sender, EventArgs e)
{
pnlPreview.Visible = true;
string previewText = txtNewPage.Text;
int maxPageLength = Convert.ToInt32(ConfigurationManager.AppSettings["maxPageLength"]);
if (maxPageLength > 0)
{
if (previewText.Length > maxPageLength)
{
previewText = "! Warning - Results truncated to not exceed " + maxPageLength.ToString() + " bytes.\r\n" +
previewText.Substring(0, maxPageLength);
}
}
litPreview.Text = WikiParser.GenerateWikiText(previewText);
}
static void Main(string[] args)
{
string[] URLS = new string[10];
URLS[0] = @"https://ru.wikipedia.org/wiki/Чужой_(фильм)";
URLS[1] = @"https://ru.wikipedia.org/wiki/Тельма_и_Луиза";
URLS[2] = @"https://ru.wikipedia.org/wiki/Версия_Браунинга_(фильм)";
URLS[3] = @"https://ru.wikipedia.org/wiki/Неприятности_с_обезьянкой";
URLS[4] = @"https://ru.wikipedia.org/wiki/Хороший_год_(фильм)";
URLS[5] = @"https://ru.wikipedia.org/wiki/Обличитель";
URLS[6] = @"https://ru.wikipedia.org/wiki/Верные_друзья_(фильм)";
URLS[7] = @"https://ru.wikipedia.org/wiki/Красная_палатка_(фильм)";
URLS[8] = @"https://ru.wikipedia.org/wiki/Летят_журавли";
URLS[9] = @"https://ru.wikipedia.org/wiki/Четыре_комнаты";
string URL = "";
string ID = "";
string Name = "";
string Year = "";
string Genre = "";
string Director = "";
for (int q = 0; q < 10; q++)
{
WikiParser.GetAllData(URLS[q], ref ID, ref URL, ref Name, ref Year, ref Genre, ref Director);
Console.WriteLine(URL);
Console.WriteLine("***");
Console.WriteLine(ID);
Console.WriteLine("***");
Console.WriteLine(Name);
Console.WriteLine("***");
Console.WriteLine(Year);
Console.WriteLine("***");
Console.WriteLine(Genre);
Console.WriteLine("***");
Console.WriteLine(Director);
Console.WriteLine("---------------------------------------------------------------");
}
Console.ReadLine();
}
protected void btnPreview_Click(object sender, EventArgs e)
{
pnlPreview.Visible = true;
//truncate if over max length
int maxPageLength = Convert.ToInt32(ConfigurationManager.AppSettings["maxPageLength"]);
string previewText = ((TextBox)DataList1.Items[0].FindControl("txtPageText")).Text;
if (maxPageLength > 0)
{
if (previewText.Length > maxPageLength)
{
previewText = "! Warning - Results truncated to not exceed " + maxPageLength.ToString() + " bytes.\r\n" +
previewText.Substring(0, maxPageLength);
}
}
litPreview.Text = WikiParser.GenerateWikiText(previewText);
}
protected void grdHistory_SelectedIndexChanged(object sender, EventArgs e)
{
pnlPageText.Visible = true;
//TODO: someone can currently promote the current page - needed if promoting a deleted page
// but otherwise will create a redundant version.
if (User.IsInRole("Editor") || User.IsInRole("Administrator"))
{
btnPromote.Visible = true;
}
else
{
btnPromote.Visible = false;
}
using (SqlConnection conn = new SqlConnection(
ConfigurationManager.ConnectionStrings["ConnectionString"].ConnectionString))
{
SqlCommand cmd = new SqlCommand("w_GetPageByVersion", conn);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.Add("@PageName", SqlDbType.VarChar).Value = pageName;
int version = int.Parse(((GridView)sender).SelectedRow.Cells[0].Text);
cmd.Parameters.Add("@Version", SqlDbType.Int).Value = version;
SqlDataReader reader;
try
{
conn.Open();
reader = cmd.ExecuteReader();
reader.Read();
lblVersionHeader.Text = string.Format("<br><H2>Page History for {0}, v{1}</H2><br>", pageName, version);
litPageHistoryView.Text = WikiParser.GenerateWikiText(reader["PageText"].ToString());
}
catch (Exception ex)
{
HandleError(ex);
}
}
}
/// <summary>
/// Получение данных для фильма с Википедии
/// </summary>
/// <param name="WikiUrl"></param>
/// <returns></returns>
public void GetDataFromWiki(string WikiUrl)
{
string Name = this.Name;
string ImdbID = this.ImdbID;
string ImdbURL = this.ImdbURL;
string Year = this.Year;
string Genre = this.Genre;
string Director = this.Director;
string temp = WikiUrl + "";
WikiParser.GetAllData(WikiUrl, ref ImdbID, ref ImdbURL, ref Name, ref Year, ref Genre, ref Director);
this.Name = Name;
this.ImdbURL = ImdbURL;
this.ImdbID = ImdbID;
this.Year = Year;
this.Genre = Genre;
this.Director = Director;
this.WikiURL = temp;
//return true; //праду говорить легко и приятно
}
public override void SetUp()
{
base.SetUp();
var urlParser = new FakeUrlParser();
wiki = new N2.Addons.Wiki.Items.Wiki();
wiki.Name = "wiki";
((IInjectable<IUrlParser>)wiki).Set(urlParser);
article = new N2.Addons.Wiki.Items.WikiArticle();
article.Name = "existing-article";
article.SavedBy = "admin";
((IInjectable<IUrlParser>)article).Set(urlParser);
article.AddTo(wiki);
parser = new WikiParser();
var pluginFinder = mocks.Stub<IPluginFinder>();
Expect.Call(pluginFinder.GetPlugins<ITemplateRenderer>()).Return(new ITemplateRenderer[] { new FakeTemplateRenderer() });
mocks.ReplayAll();
renderer = new WikiRenderer(pluginFinder, new ThreadContext());
}
public override void SetUp()
{
base.SetUp();
var urlParser = new FakeUrlParser();
wiki = new N2.Addons.Wiki.Items.Wiki();
wiki.Name = "wiki";
((IInjectable <IUrlParser>)wiki).Set(urlParser);
article = new N2.Addons.Wiki.Items.WikiArticle();
article.Name = "existing-article";
article.SavedBy = "admin";
((IInjectable <IUrlParser>)article).Set(urlParser);
article.AddTo(wiki);
parser = new WikiParser();
var pluginFinder = mocks.Stub <IPluginFinder>();
Expect.Call(pluginFinder.GetPlugins <ITemplateRenderer>()).Return(new ITemplateRenderer[] { new FakeTemplateRenderer() });
mocks.ReplayAll();
renderer = new WikiRenderer(pluginFinder, new ThreadContext());
}
protected override void OnInit(EventArgs e)
{
if (IsNew)
{
h1.Text = CurrentArguments;
}
else
{
h1.Text = CurrentPage.Title;
txtText.Text = CurrentPage.Text;
}
txtText.EnableFreeTextArea = CurrentPage.WikiRoot.EnableFreeText;
phSubmit.Visible = cvAuthorized.IsValid = IsAuthorized;
if (!string.IsNullOrEmpty(Text))
{
WikiParser parser = Engine.Resolve <WikiParser>();
WikiRenderer renderer = Engine.Resolve <WikiRenderer>();
renderer.AddTo(parser.Parse(Text), pnlMessage, CurrentPage);
}
Register.JQuery(Page);
base.OnInit(e);
}
public void SetUp()
{
parser = new WikiParser();
}
/// <summary>
/// Parses the rendering rules and adds the parsed rules to the <see cref="Rules"/> property.
/// </summary>
public void Parse()
{
string currentMainSection = null;
string currentSubsection = null;
bool ignoreRestOfMainSection = false;
bool ignoreRestOfSubsection = false;
bool moveNext = true;
rules = new RenderingRuleSet();
while (true)
{
WikiContentType wikiContentType;
if (moveNext)
{
wikiContentType = WikiParser.Next();
}
else
{
wikiContentType = WikiParser.Context.CurrentType;
moveNext = true;
}
if (wikiContentType == WikiContentType.Eof)
{
break;
}
if (wikiContentType == WikiContentType.Section || wikiContentType == WikiContentType.Text)
{
WikiSection section = WikiParser.Context.LowestSection;
if (section == null)
{
continue;
}
if (wikiContentType == WikiContentType.Section)
{
if (section.SectionLevel == 1)
{
if (log.IsDebugEnabled)
{
log.DebugFormat("Found new main section: '{0}'", section.SectionName);
}
currentMainSection = section.SectionName;
currentSubsection = null;
ignoreRestOfMainSection = false;
ignoreRestOfSubsection = false;
}
else if (section.SectionLevel == 2)
{
currentSubsection = section.SectionName;
ignoreRestOfSubsection = false;
}
}
else if (wikiContentType == WikiContentType.Text &&
(ignoreRestOfMainSection || ignoreRestOfSubsection))
{
continue;
}
switch (currentMainSection.ToLowerInvariant())
{
case "rendering rules":
{
if (currentSubsection == null)
{
continue;
}
switch (section.SectionName.ToLowerInvariant())
{
case "options":
SkipToTableAndParseIt(ParseOption);
AssertRulesVersionCompatibility();
ignoreRestOfSubsection = true;
break;
case "points":
SkipToTableAndParseIt(ParsePointRule);
ignoreRestOfSubsection = true;
break;
case "lines":
SkipToTableAndParseIt(ParseLineRule);
ignoreRestOfSubsection = true;
break;
case "areas":
SkipToTableAndParseIt(ParseAreaRule);
ignoreRestOfSubsection = true;
break;
default:
ThrowParseError("Invalid section: '{0}'", section.SectionName);
return;
}
break;
}
case "standard garmin types":
{
if (currentSubsection == null)
{
continue;
}
switch (section.SectionName.ToLowerInvariant())
{
case "points":
SkipToTableAndParseIt((tableRowCells)
=>
ParseStandardGarminTypesDictionary(
tableRowCells,
typesRegistry.GarminPointTypesDictionary));
ignoreRestOfSubsection = true;
break;
case "lines":
SkipToTableAndParseIt((tableRowCells)
=>
ParseStandardGarminTypesDictionary(
tableRowCells,
typesRegistry.GarminLineTypesDictionary));
ignoreRestOfSubsection = true;
break;
case "areas":
SkipToTableAndParseIt((tableRowCells)
=>
ParseStandardGarminTypesDictionary(
tableRowCells,
typesRegistry.GarminAreaTypesDictionary));
ignoreRestOfSubsection = true;
break;
default:
ThrowParseError("Invalid Garmin standard types section: '{0}'",
section.SectionName);
return;
}
break;
}
case "characters conversion table":
{
SkipToTableAndParseIt(ParseCharactersConversionTable);
ignoreRestOfMainSection = true;
break;
}
case "patterns":
{
if (currentSubsection == null)
{
continue;
}
ParsePointPattern(section.SectionName);
moveNext = false;
break;
}
}
}
}
}
protected void Page_Load(object sender, EventArgs e)
{
string page = "";
int v1 = 0;
int v2 = 0;
try
{
page = Request.Params["page"].ToString();
v1 = int.Parse(Request.Params["v1"].ToString());
v2 = int.Parse(Request.Params["v2"].ToString());
}
catch
{
HandleError(new Exception("This page needs a page and version parameters passed to it. If you are " +
"reading this message now, these parameters were not properly supplied. Please return to " +
"the history view for a wiki page and select the 'Show Differences' option from there, or " +
"contact your system administrator"));
Response.End();
return;
}
lblVersion1.Text = v1.ToString();
lblVersion2.Text = v2.ToString();
lnkPage.Text = page;
//now get the page text from the wiki
string page1Text = "";
string page2Text = "";
using (SqlConnection conn = new SqlConnection(ConfigurationManager.ConnectionStrings["ConnectionString"].ConnectionString))
{
SqlCommand cmd = new SqlCommand("w_GetPageByVersion", conn);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.Add("@PageName", SqlDbType.VarChar).Value = page;
cmd.Parameters.Add("@Version", SqlDbType.Int).Value = v1;
SqlDataReader reader;
try
{
conn.Open();
reader = cmd.ExecuteReader();
if (reader.Read())
{
page1Text = reader["PageText"].ToString();
}
reader.Close();
cmd.Parameters["@Version"].Value = v2;
reader = cmd.ExecuteReader();
if (reader.Read())
{
page2Text = reader["PageText"].ToString();
}
reader.Close();
}
catch (Exception ex)
{
HandleError(ex);
}
}
//display the results
string results = HTMLDiff(page1Text, page2Text);
//HACK: handle UL and OL elements and headers
results = Regex.Replace(results, @"(<span class=text(?:added|missing)>)\s?(\++|#+|!+)", "$2$1", RegexOptions.Compiled);
results = WikiParser.GenerateWikiText(results);
litVersion1.Text = (results).Replace("<span class=textadded>",
"<span class=textadded>").Replace("</span>", "</span>").Replace("<span class=textmissing>",
"<span class=textmissing>");
}
public void SetUp()
{
parser = new WikiParser();
}
protected virtual IEnumerable<WikiToken> Parse(TextReader reader, NameValueCollection headers)
{
WikiParser parser = new WikiParser();
string src = Src;
string urlFormat = WikiWordUrlFormat;
if (src.Length > 0 && urlFormat.Length > 0)
{
string basePath = VirtualPathUtility.GetDirectory(src);
string extension = VirtualPathUtility.GetExtension(src);
parser.WikiWordResolver = delegate(string word)
{
string path = VirtualPathUtility.Combine(basePath, word + extension);
if (!HostingEnvironment.VirtualPathProvider.FileExists(path))
return null;
return new Uri(ResolveUrl(string.Format(urlFormat, word)), UriKind.Relative);
};
}
return parser.Parse(reader, headers);
}
private void ParsePointPattern(string patternName)
{
currentParsedPattern = new PatternDefinition(patternName);
SkipToTableAndParseIt(ParseColorTable);
// collect all pattern lines
StringBuilder patternText = new StringBuilder();
while (true)
{
WikiContentType wikiContentType = WikiParser.Next();
if (wikiContentType != WikiContentType.Text)
{
break;
}
patternText.Append(WikiParser.Context.CurrentLine);
}
MatchCollection matches = regexPointPattern.Matches(patternText.ToString());
if (matches.Count < 1)
{
ThrowParseError("Invalid point pattern: '{0}'", patternText.ToString());
}
Match match = matches[0];
Group group = match.Groups["line"];
foreach (Capture capture in group.Captures)
{
string patternLine = capture.Value;
if (patternLine != null)
{
// all pattern lines have to be of the same length
if (currentParsedPattern.Width != -1)
{
if (patternLine.Length != currentParsedPattern.Width)
{
ThrowParseError("All pattern lines have to be of the same width.");
}
}
currentParsedPattern.PatternLines.Add(patternLine);
if (currentParsedPattern.Width > 24)
{
ThrowParseError("Point pattern can have a maximum width of 24 characters");
}
}
}
if (currentParsedPattern.Height > 24)
{
ThrowParseError("Point pattern can have a maximum height of 24 characters");
}
this.typesRegistry.Patterns.Add(currentParsedPattern.PatternName, currentParsedPattern);
}
public static void ProcessWikicode(string s, ICarnoServiceSink sink)
{
List <WikiNode> items = WikiParser.Parse(s).ToList();
var templates = from item in items
where item is WikiTemplateNode
select item as WikiTemplateNode;
foreach (WikiTemplateNode template in templates)
{
if (template.Name == "MatchList")
{
int matchno = 1;
while (true)
{
List <WikiNode> contents;
if (!template.Params.TryGetValue("match" + matchno.ToString(), out contents))
{
break;
}
if (contents.Count == 1 && contents[0] is WikiTemplateNode)
{
TryProcessMatchMaps(sink, contents[0] as WikiTemplateNode);
}
matchno++;
}
//sink.ClearTemporaryIdMap();
continue;
}
if (TryProcessMatchMaps(sink, template))
{
continue;
}
if (TryProcessBracket(sink, template))
{
continue;
}
if (TryProcessTeamBracket(sink, template))
{
continue;
}
if (TryProcessGroupTableSlot(sink, template))
{
continue;
}
if (TryProcessPlayerCrossTable(sink, template))
{
continue;
}
// if we ended up here, we don't support this template
//sw.WriteLine("; -- Unsupported template: {0} --", template.Name);
sink.UnknownTemplate(template);
}
}
void ParseWikiArticles()
{
Console.Write("Loading articles...");
XmlSerializer serializer = new XmlSerializer(typeof(Article[]),
new XmlRootAttribute() { ElementName = "Articles" });
StreamReader reader = File.OpenText("articles.xml");
articles = ((Article[])serializer.Deserialize(reader)).
OrderBy(a => a.PageId).ToDictionary(a => a.PageId, a => a);
Console.WriteLine(" Done");
Console.WriteLine(" Articles: " + articles.Count);
Console.WriteLine();
int processed = 0;
int lexerErrors = 0;
int parserErrors = 0;
fragments = new List<Fragment>();
if (File.Exists("fragments.xml"))
{
Console.Write("Loading fragments...");
serializer = new XmlSerializer(typeof(Fragment[]),
new XmlRootAttribute() { ElementName = "Fragments" });
reader = File.OpenText("fragments.xml");
fragments = ((Fragment[])serializer.Deserialize(reader)).ToList();
Console.WriteLine(" Done");
Console.WriteLine(" Fragments: " + fragments.Count);
Console.WriteLine();
}
else
{
Console.Write("Parsing articles");
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
Parallel.ForEach(articles, kv =>
//Parallel.ForEach(articles.Skip(2000).Take(1000), kv =>
{
Article article = kv.Value;
AntlrErrorListener ael = new AntlrErrorListener();
AntlrInputStream inputStream = new AntlrInputStream(article.WikiText);
WikiLexer lexer = new WikiLexer(inputStream);
lexer.RemoveErrorListeners();
lexer.AddErrorListener(ael);
CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
WikiParser parser = new WikiParser(commonTokenStream);
parser.RemoveErrorListeners();
parser.AddErrorListener(ael);
WikiParser.InitContext initContext = parser.init();
WikiVisitor visitor = new WikiVisitor(article.PageId);
visitor.VisitInit(initContext);
article.Errors = ael.ErrorList;
lock (fragments)
fragments.AddRange(visitor.Fragments);
Interlocked.Add(ref lexerErrors, ael.LexerErrors);
Interlocked.Add(ref parserErrors, ael.ParserErrors);
if (Interlocked.Increment(ref processed) % 50 == 0)
Console.Write('.');
});
stopwatch.Stop();
StreamWriter writer = File.CreateText("fragments.xml");
serializer = new XmlSerializer(typeof(Fragment[]),
new XmlRootAttribute() { ElementName = "Fragments" });
serializer.Serialize(writer, fragments.ToArray());
writer.Close();
Console.WriteLine(" Done");
Console.WriteLine(" Fragments: " + fragments.Count);
Console.WriteLine(" Parser errors: " + parserErrors);
Console.WriteLine(" Lexer errors: " + lexerErrors);
Console.WriteLine(" Parsing time: " + stopwatch.Elapsed.TotalSeconds + " sec");
Console.WriteLine();
List<string> errorLog = new List<string>();
foreach (Article article in articles.Values)
{
if (article.Errors == null)
continue;
if (article.Errors.Count == 0)
continue;
errorLog.Add("Статья: " + article.Title);
errorLog.AddRange(article.Errors);
}
File.WriteAllLines("error_log.txt", errorLog.ToArray(), Encoding.UTF8);
}
}