public static async Task Main(string[] args)
{
try
{
Regex.CacheSize = 100;
OutputDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location), "..", "..", "..", "..", "..", "Presets");
ParseArguments(args);
//Console.WriteLine($"{nameof(ForceDownload)}: {ForceDownload}");
if (!Directory.Exists(OutputDirectory))
{
Directory.CreateDirectory(OutputDirectory);
}
logger.Info($"Directory for output: \"{Path.GetFullPath(OutputDirectory)}\"");
logger.Info($"Version of preset file: {PresetVersion}");
Console.WriteLine($"Directory for output: \"{Path.GetFullPath(OutputDirectory)}\"");
Console.WriteLine($"Version of preset file: {PresetVersion}");
Console.WriteLine();
WikiTextTableContainer tableContainer = null;
if (!File.Exists("wiki_basic_info.json") || ForceDownload)
{
logger.Trace("start download of basic wiki building info");
var provider = new WikiTextProvider();
var providerResult = await provider.GetWikiTextAsync();
var tableParser = new WikiTextTableParser();
tableContainer = tableParser.GetTables(providerResult.WikiText);
logger.Trace("save basic wiki building info");
SerializationHelper.SaveToFile(tableContainer, "wiki_basic_info.json", prettyPrint: true);
}
else
{
logger.Trace("load basic wiki building info");
tableContainer = SerializationHelper.LoadFromFile <WikiTextTableContainer>("wiki_basic_info.json");
}
//get basic info of all buildings
var wikiBuildingInfoProvider = new WikiBuildingInfoProvider();
var wikiBuildingInfoPreset = wikiBuildingInfoProvider.GetWikiBuildingInfos(tableContainer);
//get detail info of all buildings
var specialBuildingNameHelper = new SpecialBuildingNameHelper();
var regionHelper = new RegionHelper();
var titleParserSingle = new TitleParserSingle(_commons, specialBuildingNameHelper);
var infoboxParser = new InfoboxParser.InfoboxParser(_commons, titleParserSingle, specialBuildingNameHelper, regionHelper);
var infoboxExtractor = new InfoboxExtractor(_commons, titleParserSingle);
var wikiDetailProvider = new WikiBuildingDetailProvider(_commons, infoboxParser, infoboxExtractor);
wikiBuildingInfoPreset = wikiDetailProvider.FetchBuildingDetails(wikiBuildingInfoPreset);
wikiBuildingInfoPreset.Version = PresetVersion;
wikiBuildingInfoPreset.DateGenerated = DateTime.UtcNow;
var outputPath = Path.Combine(OutputDirectory, CoreConstants.PresetsFiles.WikiBuildingInfoPresetsFile);
logger.Trace($"save wiki building info: {outputPath}");
SerializationHelper.SaveToFile(wikiBuildingInfoPreset, outputPath, prettyPrint: UsePrettyPrint);
//for testing
//load parsed file to test
//wikiBuildingInfoPreset = SerializationHelper.LoadFromFile<WikiBuildingInfoPreset>(PRESET_FILENAME);
var oldColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"Finished parsing all building information.");
Console.ForegroundColor = oldColor;
}
catch (Exception ex)
{
logger.Error(ex, "error occured");
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(ex);
}
finally
{
if (!NoWait)
{
Console.WriteLine();
Console.WriteLine("To exit press any key ...");
Console.ReadLine();
}
}
}
private void SaveCompleteInfos(WikiBuildingInfoPresets wikiBuildingInfoList)
{
Console.WriteLine("start fetching building details");
if (!Directory.Exists(PathToDetailsFolder))
{
Directory.CreateDirectory(PathToDetailsFolder);
}
var existingMissingInfos = new List <string>();
if (File.Exists(Path.Combine(PathToDetailsFolder, FILENAME_MISSING_INFOS)))
{
existingMissingInfos.AddRange(File.ReadAllLines(Path.Combine(PathToDetailsFolder, FILENAME_MISSING_INFOS), Encoding.UTF8));
}
var missingInfos = new List <string>();
Stopwatch sw = new Stopwatch();
sw.Start();
Parallel.ForEach(wikiBuildingInfoList.Infos,
new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
},
curBuilding =>
{
var pageName = curBuilding.Name.Replace(" ", "_");
//only download when not present and not forced to download
var fileName = GetCleanedFilename(pageName);
var destinationFilePath = Path.Combine(PathToDetailsFolder, $"{fileName}{FILE_ENDING_WIKITEXT}");
if (!Program.ForceDownload && (File.Exists(destinationFilePath) || existingMissingInfos.Contains(curBuilding.Name)))
{
return;
}
var provider = new WikiTextProvider();
var providerResult = provider.GetWikiTextAsync(pageName).GetAwaiter().GetResult();
if (!string.IsNullOrWhiteSpace(providerResult.WikiText))
{
providerResult.WikiText = GetLineBreakAlignedWikiText(providerResult.WikiText);
//align infobox name
providerResult.WikiText = providerResult.WikiText.Replace("{{Infobox_Buildings", "{{Infobox Buildings");
File.WriteAllText(destinationFilePath, providerResult.WikiText, Encoding.UTF8);
File.AppendAllLines(destinationFilePath, new List <string>
{
Environment.NewLine,
$"{REVISION_HEADER_ID}{REVISION_SEPARATOR}{providerResult.RevisionId}",
$"{REVISION_HEADER_DATE}{REVISION_SEPARATOR}{providerResult.EditDate.ToString("o")}"
}, Encoding.UTF8);
}
else
{
missingInfos.Add(curBuilding.Name);
}
});
if (missingInfos.Count > 0)
{
File.WriteAllLines(Path.Combine(PathToDetailsFolder, FILENAME_MISSING_INFOS), missingInfos.Distinct().OrderBy(x => x), Encoding.UTF8);
}
sw.Stop();
var message = $"finished fetching building details (took {sw.ElapsedMilliseconds} ms)";
logger.Trace(message);
Console.WriteLine(message);
}
