private static void GenerateDefaultOutput(
string input,
string output,
Dictionary <string, List <string> > features,
List <string> filters,
Dictionary <string, List <string> > countries,
Dictionary <string, List <string> > admin1codes,
Dictionary <string, List <string> > admin2codes)
{
// check input file if it exists
if (!File.Exists(input))
{
Console.WriteLine("Input file not found");
return;
}
// check output file if it exists
if (File.Exists(output))
{
File.Delete(output);
}
// read input file
try
{
using StreamReader sr = new StreamReader(input);
using StreamWriter sw = new StreamWriter(output);
sw.Write("[");
string inputLine = string.Empty;
bool isfirst = true;
var options = new JsonSerializerOptions()
{
IgnoreNullValues = true
};
while ((inputLine = sr.ReadLine()) != null)
{
var s = inputLine.Split(new char[] { '\t' });
// check if feature is included in filters
var featurecode = $"{s[6]}.{s[7]}";
if (filters.FirstOrDefault(r => r.Equals(featurecode, StringComparison.OrdinalIgnoreCase)) == null)
{
continue;
}
var geo = new geonameFlat();
if (long.TryParse(s[0], out long geoid))
{
geo.geonameId = geoid;
}
geo.name = s[1].Trim();
geo.asciiName = s[2].Trim();
var altNames = s[3].Split(new char[] { ',' });
geo.alternateNames = string.IsNullOrWhiteSpace(altNames[0]) ? null : altNames;
if (decimal.TryParse(s[4], out decimal lat))
{
geo.latitude = lat;
}
if (decimal.TryParse(s[5], out decimal lng))
{
geo.longitude = lng;
}
if (features.TryGetValue(featurecode, out List <string> feats))
{
geo.feature = feats[0];
}
geo.countryCode = s[8];
if (countries.TryGetValue(geo.countryCode, out List <string> ctry))
{
geo.countryCodeISO3 = ctry[0];
geo.country = ctry[3];
geo.continent = ctry[7];
geo.tld = ctry[8];
geo.currencyCode = ctry[9];
geo.currency = ctry[10];
var langs = ctry[14].Split(new char[] { ',' });
geo.languages = string.IsNullOrWhiteSpace(langs[0]) ? null : langs;
var neighbors = ctry[16].Split(new char[] { ',' });
geo.neighborCountries = string.IsNullOrWhiteSpace(neighbors[0]) ? null : neighbors;
}
var admin1code = $"{geo.countryCode}.{s[10]}";
if (admin1codes.TryGetValue(admin1code, out List <string> a1))
{
geo.admin1 = a1[0].Trim();
}
var admin2code = $"{admin1code}.{s[11]}";
if (admin2codes.TryGetValue(admin2code, out List <string> a2))
{
geo.admin2 = a2[0].Trim();
}
geo.timezone = s[17];
if (!string.IsNullOrWhiteSpace(geo.timezone))
{
geo.windowsTimezone = TZConvert.IanaToWindows(geo.timezone);
}
geo.dateModified = DateTime.Parse(s[18]);
var towrite = JsonSerializer.Serialize(geo, options);
if (isfirst)
{
isfirst = false;
}
else
{
sw.Write(',');
}
sw.Write(towrite);
}
sw.Write("]");
sw.Flush();
}
catch (IOException e)
{
Console.WriteLine(e.Message);
}
}
private static void GenerateFullStructuredOutput(
string input,
string output,
Dictionary <string, List <string> > features,
List <string> filters,
Dictionary <string, List <string> > countries,
Dictionary <string, List <string> > admin1codes,
Dictionary <string, List <string> > admin2codes)
{
Console.WriteLine("Start GenerateFullStructuredOutput");
// check input file if it exists
if (!File.Exists(input))
{
Console.WriteLine("Input file not found");
return;
}
// check output file if it exists
if (File.Exists(output))
{
File.Delete(output);
}
// read input file
try
{
using StreamReader sr = new StreamReader(input);
using StreamWriter sw = new StreamWriter(output);
var semaphore = new Semaphore(1000, 1000);
object filelock = new object();
string inputLine = string.Empty;
var options = new JsonSerializerOptions()
{
IgnoreNullValues = true
};
var tracker = new Dictionary <string, geonameStructured>();
var tasks = new List <Task>();
while ((inputLine = sr.ReadLine()) != null)
{
semaphore.WaitOne();
var s = inputLine.Split(new char[] { '\t' });
var handle = new EventWaitHandle(false, EventResetMode.ManualReset);
var task = Task.Run(action: () =>
{
Console.WriteLine("Processing GeonameID: " + s[0]);
var geo = new geonameFlat();
// check if feature is included in filters
geo.feature = $"{s[6]}.{s[7]}";
if (filters.FirstOrDefault(r => r.Equals(geo.feature, StringComparison.OrdinalIgnoreCase)) == null)
{
return;
}
geo.countryCode = s[8];
var admin1code = $"{geo.countryCode}.{s[10]}";
var admin2code = $"{admin1code}.{s[11]}";
geonameStructured geostruct = null;
var key = $"{geo.countryCode}-{admin1code}-{admin2code}";
lock (filelock)
{
if (!tracker.TryGetValue(key, out geostruct))
{
geostruct = new geonameStructured()
{
countryCode = geo.countryCode,
admin1Code = admin1code,
admin2Code = admin2code,
tempFilename = Path.GetTempFileName(),
isNewTempFile = true
};
if (countries.TryGetValue(geo.countryCode, out List <string> ctry))
{
geostruct.countryCodeISO3 = ctry[0];
geostruct.country = ctry[3];
geostruct.continent = ctry[7];
geostruct.tld = ctry[8];
geostruct.currencyCode = ctry[9];
geostruct.currency = ctry[10];
geostruct.currency = ctry[10];
var langs = ctry[14].Split(new char[] { ',' });
geostruct.languages = string.IsNullOrWhiteSpace(langs[0]) ? null : langs;
var neighbors = ctry[16].Split(new char[] { ',' });
geostruct.neighborCountries = string.IsNullOrWhiteSpace(neighbors[0]) ? null : neighbors;
}
if (admin1codes.TryGetValue(admin1code, out List <string> a1))
{
geostruct.admin1 = a1[0] ?? "None";
}
if (admin2codes.TryGetValue(admin2code, out List <string> a2))
{
geostruct.admin2 = a2[0] ?? "None";
}
tracker.Add(geostruct.key, geostruct);
File.WriteAllText(geostruct.tempFilename, "[");
}
}
if (long.TryParse(s[0], out long geoid))
{
geo.geonameId = geoid;
}
geo.name = s[1].Trim();
geo.asciiName = s[2].Trim();
var altNames = s[3].Split(new char[] { ',' });
geo.alternateNames = string.IsNullOrWhiteSpace(altNames[0]) ? null : altNames;
if (decimal.TryParse(s[4], out decimal lat))
{
geo.latitude = lat;
}
if (decimal.TryParse(s[5], out decimal lng))
{
geo.longitude = lng;
}
//if (countries.TryGetValue(geo.countryCode, out List<string> ctry))
//{
// geo.countryCodeISO3 = ctry[0];
// geo.country = ctry[3];
// geo.continent = ctry[7];
// geo.tld = ctry[8];
// geo.currencyCode = ctry[9];
// geo.currency = ctry[10];
// var langs = ctry[14].Split(new char[] { ',' });
// geo.languages = string.IsNullOrWhiteSpace(langs[0]) ? null : langs;
// var neighbors = ctry[16].Split(new char[] { ',' });
// geo.neighborCountries = string.IsNullOrWhiteSpace(neighbors[0]) ? null : neighbors;
//}
//if (admin1codes.TryGetValue(admin1code, out List<string> a1))
//{
// geo.admin1 = a1[0].Trim();
//}
//if (admin2codes.TryGetValue(admin2code, out List<string> a2))
//{
// geo.admin2 = a2[0].Trim();
//}
geo.timezone = s[17];
if (!string.IsNullOrWhiteSpace(geo.timezone))
{
geo.windowsTimezone = TZConvert.IanaToWindows(geo.timezone);
}
geo.dateModified = DateTime.Parse(s[18]);
lock (geostruct.filelock)
{
if (!geostruct.isNewTempFile)
{
File.AppendAllText(geostruct.tempFilename, ",");
}
geostruct.isNewTempFile = false;
File.AppendAllText(geostruct.tempFilename, JsonSerializer.Serialize(geo, options));
}
});
tasks.Add(task);
semaphore.Release();
Console.WriteLine("--- Done GeonameID: " + s[0]);
}
Console.WriteLine("Waiting Tasks");
Task.WaitAll(tasks.ToArray());
// close array
Console.WriteLine("Closing Array in Files");
foreach (var t in tracker)
{
File.AppendAllText(t.Value.tempFilename, "]");
}
Console.WriteLine("Flatten Tracker");
// flatten tracker
List <geonameStructured> flattracker = new List <geonameStructured>();
foreach (var f in tracker)
{
flattracker.Add(f.Value);
}
// Write output
sw.Write("[{");
var ctrys = flattracker.GroupBy(r => r.countryCode).Select(r => r.First());
bool isFirstCountry = true;
foreach (var ctry in ctrys)
{
if (!isFirstCountry)
{
sw.Write(",");
}
isFirstCountry = false;
sw.Write($"\"{ctry.country ?? "None"}\":{{");
sw.Write($"\"CountryCode\":\"{ctry.countryCode}\",");
sw.Write($"\"CountryISO\":\"{ctry.countryCodeISO3}\",");
sw.Write($"\"Continent\":\"{ctry.continent}\",");
sw.Write($"\"Currency\":\"{ctry.currency}\",");
sw.Write($"\"CurrencyCode\":\"{ctry.currencyCode}\",");
sw.Write($"\"Languages\":{JsonSerializer.Serialize(ctry.languages, options)},");
sw.Write($"\"Neighbors\":{JsonSerializer.Serialize(ctry.neighborCountries, options)},");
sw.Write($"\"TLD\":\"{ctry.tld}\",");
sw.Write("\"Regions\":[{");
var admin1s = flattracker.Where(r => r.countryCode.Equals(ctry.countryCode, StringComparison.OrdinalIgnoreCase)).GroupBy(r => r.admin1Code).Select(r => r.First());
bool isFirstAdmin1 = true;
foreach (var a1 in admin1s)
{
if (!isFirstAdmin1)
{
sw.Write(",");
}
isFirstAdmin1 = false;
sw.Write($"\"{a1.admin1 ?? "None"}\":{{");
var admin2s = flattracker.Where(r => r.admin1Code.Equals(a1.admin1Code, StringComparison.OrdinalIgnoreCase)).GroupBy(r => r.admin2Code).Select(r => r.First());
bool isFirstAdmin2 = true;
foreach (var a2 in admin2s)
{
if (!isFirstAdmin2)
{
sw.Write(",");
}
isFirstAdmin2 = false;
sw.Write($"\"{a2.admin2 ?? "None"}\":");
var files = flattracker.Where(r => r.admin2Code.Equals(a2.admin2Code, StringComparison.OrdinalIgnoreCase));
bool isFirstFile = true;
foreach (var f in files)
{
var content = File.ReadAllText(f.tempFilename);
if (!isFirstFile)
{
sw.Write(",");
}
isFirstFile = false;
sw.Write(content);
}
}
sw.Write("}");
}
sw.Write("}]}");
}
sw.Write("}]");
sw.Flush();
Console.WriteLine("Output Done");
// delete tracker temp files
foreach (var t in tracker)
{
if (File.Exists(t.Value.tempFilename))
{
File.Delete(t.Value.tempFilename);
}
}
Console.WriteLine("Delete Temp Files");
}
catch (IOException e)
{
Console.WriteLine(e.Message);
}
}
