public void ReverseGeoCode_RadialSearch_ReturnsMaxCountResults()
{
// Read a file with data with points in and around London in a 20Km radius
var data = GeoFileReader.ReadExtendedGeoNames(@"testdata\test_GB.txt").ToArray();
var rg = new ReverseGeoCode <ExtendedGeoName>(data);
var center = rg.CreateFromLatLong(51.5286416, 0); //Exactly at 0 longitude so we test left/right of prime meridian
var maxresults = 10;
Assert.AreEqual(47, data.Length); //File should contain 47 records total
var expected_ids = new[] { 2643741, 2646003, 2643743, 6690870, 2651621, 2655775, 2636503, 2634677, 2656194, 2653266 };
Assert.AreEqual(maxresults, expected_ids.Length);
// Search from the/a center in London for all points in a 10Km radius, allowing only maxresults results
var searchresults = rg.RadialSearch(center, 100000.0, maxresults).ToArray();
// Number of results should match length of expected_id array
Assert.AreEqual(expected_ids.Length, searchresults.Length);
// Check if each result is in the expected results array
foreach (var r in searchresults)
{
Assert.IsTrue(expected_ids.Contains(r.Id));
}
}
void EnsureLoaded()
{
if (this.geodata == null)
{
lock (this.syncLock)
{
if (this.geodata == null)
{
using (var stream = this.LoadStream("admin1CodesASCII.txt"))
{
this.adminCodes = GeoFileReader
.ReadAdmin1Codes(stream)
.ToList();
}
using (var stream = this.LoadStream("cities5000.txt"))
{
this.geodata = GeoFileReader
.ReadExtendedGeoNames(stream)
.Where(x =>
//x.FeatureCode.Equals("PPL", StringComparison.OrdinalIgnoreCase) &&
//(
x.CountryCode.Equals("CA", StringComparison.CurrentCultureIgnoreCase) ||
x.CountryCode.Equals("US", StringComparison.CurrentCultureIgnoreCase)
//)
)
.OrderBy(x => x.Latitude)
.ThenBy(x => x.Longitude)
.ToList();
}
}
}
}
}
public void ExtendedGeoNamesComposer_ComposesFileCorrectly()
{
var src = @"testdata\test_extendedgeonames.txt";
var dst = @"testdata\test_extendedgeonames.out.txt";
GeoFileWriter.WriteExtendedGeoNames(dst, GeoFileReader.ReadExtendedGeoNames(src));
FileUtil.EnsureFilesAreFunctionallyEqual(src, dst, 19, 0, new[] { '\t' }, Encoding.UTF8, false);
}
public void Composer_HandlesGZippedFilesCorrectly()
{
var src = @"testdata\test_extendedgeonames.txt";
var dst = @"testdata\test_extendedgeonames.out.gz";
GeoFileWriter.WriteExtendedGeoNames(dst, GeoFileReader.ReadExtendedGeoNames(src));
Assert.AreEqual(7, GeoFileReader.ReadExtendedGeoNames(dst).Count());
}
public void FileReader_HandlesEmptyFilesCorrectly()
{
//Could use ANY entity
var target1 = new GeoFileReader().ReadRecords <CustomEntity>(@"testdata\emptyfile.txt", new CustomParser(19, 5, new[] { '☃' }, Encoding.UTF7, true)).ToArray();
Assert.AreEqual(0, target1.Length);
//Here's a second one
var target2 = GeoFileReader.ReadExtendedGeoNames(@"testdata\emptyfile.txt").ToArray();
Assert.AreEqual(0, target2.Length);
}
private static void DumpASCIILies(string logpath)
{
using (var lw = File.CreateText(Path.Combine(logpath, "_asciilies.log")))
{
//Test for fields that claim to contain ASCII only but contain non-ASCII data anyways
var nonasciifilter = new Regex("[^\x20-\x7F]", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
var geofilefilter = new Regex("^[A-Z]{2}.txt$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
lw.WriteLine("The following files contain entries that claim to contain ASCII only but contain non-ASCII data anyways:");
var extgeofiles = new[] { "allCountries", "cities1000", "cities5000", "cities15000", "no-country" }
.Select(f => Path.Combine(Dump_DownloadDirectory, f + ".txt"))
.Union(Directory.GetFiles(Dump_DownloadDirectory, "*.txt")
.Where(f => geofilefilter.IsMatch(Path.GetFileName(f)))
);
var lies = extgeofiles.AsParallel()
.SelectMany(f => GeoFileReader.ReadExtendedGeoNames(f)
.Where(e => nonasciifilter.IsMatch(e.NameASCII))
.Select(i => new NonASCIIEntry {
FileName = f, Id = i.Id, Value = i.NameASCII
})
).Union(
GeoFileReader.ReadAdmin1Codes(Path.Combine(Dump_DownloadDirectory, "admin1CodesASCII.txt")).AsParallel()
.Where(c => nonasciifilter.IsMatch(c.NameASCII))
.Select(i => new NonASCIIEntry {
FileName = "admin1CodesASCII.txt", Id = i.GeoNameId, Value = i.NameASCII
})
).Union(
GeoFileReader.ReadAdmin2Codes(Path.Combine(Dump_DownloadDirectory, "admin2Codes.txt")).AsParallel()
.Where(c => nonasciifilter.IsMatch(c.NameASCII))
.Select(i => new NonASCIIEntry {
FileName = "admin2Codes.txt", Id = i.GeoNameId, Value = i.NameASCII
})
);
foreach (var l in lies.OrderBy(l => l.FileName).ThenBy(l => l.Value))
{
lw.WriteLine(string.Join("\t", Path.GetFileName(l.FileName), l.Id, l.Value));
}
;
}
}
private void InitReverseGeoCode()
{
if (_reverseGeoCode != null)
{
return;
}
try
{
_reverseGeoCode = new ReverseGeoCode <ExtendedGeoName>(
GeoFileReader.ReadExtendedGeoNames(
Path.Combine(_appSettings.TempFolder, GeoFileDownload.CountryName + ".txt")));
}
catch (ParserException e)
{
_logger?.LogError(e, "catch-ed GeoFileDownload GeoReverseLookup error");
}
catch (FileNotFoundException e)
{
_logger?.LogError(e, "catch-ed GeoFileDownload GeoReverseLookup error");
}
}
public void ReverseGeoCode_NearestNeighbourSearch_ReturnsCorrectResults()
{
// Read a file with data with points in and around London in a 20Km radius
var data = GeoFileReader.ReadExtendedGeoNames(@"testdata\test_GB.txt").ToArray();
var rg = new ReverseGeoCode <ExtendedGeoName>(data);
var center = rg.CreateFromLatLong(51.5286416, 0); //Exactly at 0 longitude so we test left/right of prime meridian
Assert.AreEqual(47, data.Length); //File should contain 47 records total
var expected_ids = new[] { 2640729, 2639577, 2642465, 2637627, 2633709, 2643339, 2634677, 2636503, 2652053, 2654710, 2643743, 2646003, 2643741, 2653941, 6690870, 2655775, 2651621, 2650497, 2656194, 2653266, 2648657, 2637433, 2652618, 2646057 };
// Search from the/a center in London for the first X points (where X == expected_ids.length)
var searchresults = rg.NearestNeighbourSearch(center, expected_ids.Length).ToArray();
// Number of results should match length of expected_id array
Assert.AreEqual(expected_ids.Length, searchresults.Length);
// Check if each result is in the expected results array
foreach (var r in searchresults)
{
Assert.IsTrue(expected_ids.Contains(r.Id));
}
}
public static void Populate(IConfiguration configuration)
{
Console.WriteLine("Getting ready to populate countries…");
var connectionString = configuration["ConnectionString"];
var dataPath = configuration["DataSourcePath"];
var countriesPath = Path.Combine(dataPath, @"allCountries.txt");
if (!File.Exists(countriesPath))
{
Console.WriteLine("Downloading GeoNames data for all countries…");
var downloader = GeoFileDownloader.CreateGeoFileDownloader();
downloader.DownloadFile("allCountries.zip", dataPath);
}
var results = GeoFileReader.ReadExtendedGeoNames(countriesPath).OrderBy(p => p.Id);
using (var connection = new SqliteConnection(connectionString))
{
connection.Open();
Console.WriteLine("Populating countries…");
var allowIdentityInsert = connection.CreateCommand();
allowIdentityInsert.CommandText = @"SET IDENTITY_INSERT GeoNames ON";
try
{
allowIdentityInsert.ExecuteNonQuery();
}
catch (SqliteException exception)
{
Console.WriteLine("SQL Exception occurred. Error Code: " + exception.ErrorCode);
}
const string sql = @"INSERT INTO GeoNames (ID, Name, NameASCII, Latitude, Longitude, FeatureClass, FeatureCode, CountryCode,
Population, Elevation, Dem, Timezone, ModificationDate)
VALUES (@ID, @Name, @NameASCII, @Latitude, @Longitude, @FeatureClass, @FeatureCode, @CountryCode, @Population, @Elevation,
@Dem, @Timezone, @ModificationDate)";
var command = connection.CreateCommand();
command.CommandText = sql;
string[] parameterNames = new[]
{
"@ID",
"@Name",
"@NameASCII",
"@Latitude",
"@Longitude",
"@FeatureClass",
"@FeatureCode",
"@CountryCode",
"@Population",
"@Elevation",
"@Dem",
"@Timezone",
"@ModificationDate"
};
DbParameter[] parameters = parameterNames.Select(pn =>
{
DbParameter parameter = command.CreateParameter();
parameter.ParameterName = pn;
command.Parameters.Add(parameter);
return(parameter);
})
.ToArray();
foreach (var r in results)
{
parameters[0].Value = r.Id;
parameters[1].Value = r.Name.HasValueOrDBNull();
parameters[2].Value = r.NameASCII.HasValueOrDBNull();
parameters[3].Value = r.Latitude;
parameters[4].Value = r.Longitude;
parameters[5].Value = r.FeatureClass.HasValueOrDBNull();
parameters[6].Value = r.FeatureCode.HasValueOrDBNull();
parameters[7].Value = r.CountryCode.HasValueOrDBNull();
parameters[8].Value = r.Population;
parameters[9].Value = r.Elevation.HasValueOrDBNull();
parameters[10].Value = r.Dem;
parameters[11].Value = r.Timezone.HasValueOrDBNull();
parameters[12].Value = r.ModificationDate;
command.ExecuteNonQuery();
Console.WriteLine("GeoName ID: " + r.Id);
}
var disallowIdentityInsert = connection.CreateCommand();
disallowIdentityInsert.CommandText = @"SET IDENTITY_INSERT GeoNames OFF";
try
{
disallowIdentityInsert.ExecuteNonQuery();
}
catch (SqliteException exception)
{
Console.WriteLine("SQL Exception occurred. Error Code: " + exception.ErrorCode);
}
Console.WriteLine();
}
}
public static void Populate(IConfiguration configuration)
{
Console.WriteLine("Getting ready to populate GeoNames…");
var connectionString = configuration["ConnectionString"];
var dataPath = configuration["DataSourcePath"];
var minimumPopulation = configuration["GeoNames:CitiesMinimumPopulation"];
var includeCountriesData = configuration["GeoNames:IncludeCountriesData"];
string citiesFileName;
switch (minimumPopulation)
{
case "1000":
citiesFileName = "cities1000";
break;
case "5000":
citiesFileName = "cities5000";
break;
case "15000":
citiesFileName = "cities15000";
break;
default:
citiesFileName = "cities15000";
break;
}
var citiesPath = Path.Combine(dataPath, citiesFileName + ".txt");
if (!File.Exists(citiesPath))
{
Console.WriteLine("Downloading specified GeoNames data…");
var downloader = GeoFileDownloader.CreateGeoFileDownloader();
downloader.DownloadFile(citiesFileName + ".zip", dataPath);
}
var results = GeoFileReader.ReadExtendedGeoNames(citiesPath).ToList();
if (includeCountriesData == "true")
{
var countriesPath = Path.Combine(dataPath, @"allCountries.txt");
if (!File.Exists(countriesPath))
{
Console.WriteLine("Downloading GeoNames data for all countries…");
var downloader = GeoFileDownloader.CreateGeoFileDownloader();
downloader.DownloadFile("allCountries.zip", dataPath);
}
var countriesNames = GeoFileReader.ReadExtendedGeoNames(countriesPath)
.Where(p => p.FeatureCode == "PCLI" || p.FeatureCode == "ADM1");
results.AddRange(countriesNames);
}
results.OrderBy(p => p.Id);
using (var connection = new SqliteConnection(connectionString))
{
connection.Open();
Console.WriteLine("Populating GeoNames…");
const string sql = @"INSERT INTO GeoNames (ID, Name, NameASCII, Latitude, Longitude, FeatureClass, FeatureCode, CountryCode,
Population, Elevation, Dem, Timezone, ModificationDate)
VALUES (@ID, @Name, @NameASCII, @Latitude, @Longitude, @FeatureClass, @FeatureCode, @CountryCode, @Population, @Elevation,
@Dem, @Timezone, @ModificationDate)";
var command = connection.CreateCommand();
command.CommandText = sql;
string[] parameterNames = new[]
{
"@ID",
"@Name",
"@NameASCII",
"@Latitude",
"@Longitude",
"@FeatureClass",
"@FeatureCode",
"@CountryCode",
"@Population",
"@Elevation",
"@Dem",
"@Timezone",
"@ModificationDate"
};
DbParameter[] parameters = parameterNames.Select(pn =>
{
DbParameter parameter = command.CreateParameter();
parameter.ParameterName = pn;
command.Parameters.Add(parameter);
return(parameter);
})
.ToArray();
foreach (var r in results)
{
parameters[0].Value = r.Id;
parameters[1].Value = r.Name.HasValueOrDBNull();
parameters[2].Value = r.NameASCII.HasValueOrDBNull();
parameters[3].Value = r.Latitude;
parameters[4].Value = r.Longitude;
parameters[5].Value = r.FeatureClass.HasValueOrDBNull();
parameters[6].Value = r.FeatureCode.HasValueOrDBNull();
parameters[7].Value = r.CountryCode.HasValueOrDBNull();
parameters[8].Value = r.Population;
parameters[9].Value = r.Elevation.HasValueOrDBNull();
parameters[10].Value = r.Dem;
parameters[11].Value = r.Timezone.HasValueOrDBNull();
parameters[12].Value = r.ModificationDate;
command.ExecuteNonQuery();
Console.WriteLine("GeoName ID: " + r.Id);
}
Console.WriteLine();
}
}
public void FileReader_ExtendedGeoNames_StreamOverload()
{
using (var s = File.OpenRead(@"testdata\test_extendedgeonames.txt"))
GeoFileReader.ReadExtendedGeoNames(s).Count();
}
public void ExtendedGeoNameParser_ParsesFileCorrectly()
{
var target = GeoFileReader.ReadExtendedGeoNames(@"testdata\test_extendedgeonames.txt").ToArray();
Assert.AreEqual(7, target.Length);
//No alternate names/countrycodes and empty admincodes
Assert.AreEqual(string.Empty, target[0].Admincodes[0]);
Assert.AreEqual(string.Empty, target[0].Admincodes[1]);
Assert.AreEqual(string.Empty, target[0].Admincodes[2]);
Assert.AreEqual(string.Empty, target[0].Admincodes[3]);
Assert.AreEqual(0, target[0].AlternateCountryCodes.Length);
Assert.AreEqual(0, target[0].AlternateNames.Length);
//Specified alternate names/countrycodes and specified admincodes
Assert.AreEqual("09", target[1].Admincodes[0]);
Assert.AreEqual("900", target[1].Admincodes[1]);
Assert.AreEqual("923", target[1].Admincodes[2]);
Assert.AreEqual("2771781", target[1].Admincodes[3]);
Assert.AreEqual(1, target[1].AlternateCountryCodes.Length);
Assert.AreEqual(2, target[1].AlternateNames.Length);
//Lots of alternate countrycodes/alternatenames
CollectionAssert.AreEqual(new[] { "DZ", "LY", "MR", "TN", "EG", "MA", "EH", "ML", "NE", "TD", "SD" }, target[2].AlternateCountryCodes);
#region Charset test
CollectionAssert.AreEqual(new[] {
"An Bhoisnia agus Heirseagovein",
"An Bhoisnia agus Heirseagóvéin",
"An Bhoisnia-Heirseagaivein",
"An Bhoisnia-Heirseagaivéin",
"Bhosnia le Herzegovina",
"Bo-xni-a Hec-xe-go-vi-na",
"Bo-xni-a Hec-xe-go-vi-na (Bosnia va Herzegovina)",
"Bos'nija i Gercagavina",
"Bosenia me Hesegowina",
"Bosini mpe Hezegovine",
"Bosini mpé Hezegovine",
"Bosiniya na Herigozevine",
"Bosiniya na Herizegovina",
"Bosmudin boln Khercegudin Orn",
"Bosna",
"Bosna Hersek",
"Bosna a Hercegovina",
"Bosna a Hercegowina",
"Bosna agus Hearsagobhana",
"Bosna doo Hetsog Bikeyah",
"Bosna dóó Hetsog Bikéyah",
"Bosna i Hercegovina",
"Bosna i Khercegovina",
"Bosna in Hercegovina",
"Bosna kap Hercegovina",
"Bosna u Hersek",
"Bosna va Hercegovina",
"Bosna ve Hersek",
"Bosna và Hercegovina",
"Bosna-Gercegovina",
"Bosna-Hersek",
"Bosnaen e Haerzegovaen",
"Bosneja i Hercegovina",
"Bosneje er Hercuogovena",
"Bosni",
"Bosni a Gercegovina a",
"Bosni aemae Gercegovinae",
"Bosni ak Erzegovin",
"Bosni ba Gercegovina",
"Bosni ba Khercegovina",
"Bosni tata Gercegovina",
"Bosnia",
"Bosnia & Herzegovina",
"Bosnia - Erzegobine",
"Bosnia - Hercegovina - Bosna i Khercegovina",
"Bosnia - Hercegovina - Босна и Херцеговина",
"Bosnia Ercegovina",
"Bosnia Erzegovina",
"Bosnia Herzegovina",
"Bosnia Herzogovina",
"Bosnia Hèrzègovina",
"Bosnia a Hercegovina",
"Bosnia a Herzegovina",
"Bosnia aamma Herzegovina",
"Bosnia and Hercegovina",
"Bosnia and Herzegovina",
"Bosnia as Herzegovina",
"Bosnia at Herzegovina",
"Bosnia ati Herjegofina",
"Bosnia dan Herzegovina",
"Bosnia e Ercegovina",
"Bosnia e Erzegovina",
"Bosnia e Erzegòvina",
"Bosnia e Hercegovina",
"Bosnia e Hertsegovina",
"Bosnia e Herzegovina",
"Bosnia ed Erzegovina",
"Bosnia et Herzegovina",
"Bosnia ev Hercʻegovina",
"Bosnia ha Herzegovina",
"Bosnia i Gercagavina",
"Bosnia i Gercogovina",
"Bosnia i Hercegovina",
"Bosnia i Hercegowina",
"Bosnia i Hersegovina",
"Bosnia ihuan Hertzegovina",
"Bosnia ja Hercegovina",
"Bosnia ja Hertsegoviina",
"Bosnia ja Hertsegovina",
"Bosnia ja Herzegovina",
"Bosnia jeung Herzegovina",
"Bosnia jeung Hérzégovina",
"Bosnia kai Erzegobine",
"Bosnia ken Herzegovina",
"Bosnia kple Herzergovina nutome",
"Bosnia ma Herzegovina",
"Bosnia na Erzegovina",
"Bosnia na Herzegovina",
"Bosnia na Hezegovina",
"Bosnia ne Hɛzegovina",
"Bosnia og Hercegovina",
"Bosnia og Hersegovina",
"Bosnia si Hertegovina",
"Bosnia sy Herzegovina",
"Bosnia ta Gercogovina",
"Bosnia tan Hersegobina",
"Bosnia ug Herzegovina",
"Bosnia y Hercegovina",
"Bosnia y Herzegovina",
"Bosnia è Erzegovina",
"Bosnia īhuān Hertzegovina",
"Bosnia și Herțegovina",
"Bosnia-Ercegovina",
"Bosnia-Erzegovina",
"Bosnia-Hercegovina",
"Bosnia-Hersegovina",
"Bosnia-Hertsegovina",
"Bosnia-Herzegovina",
"Bosnia-ha-Herzegovina",
"Bosnie Herzegovina",
"Bosnie an Herzegovinae",
"Bosnie en Herzegovina",
"Bosnie en Herzegowina",
"Bosnie-Erzegovine",
"Bosnie-Hercegovina",
"Bosnie-Herzegovena",
"Bosnie-Herzegovina",
"Bosnie-Herzegovine",
"Bosnie-Herzegowina",
"Bosnie-Herzégovine",
"Bosnie-Hèrzègovena",
"Bosnie-Érzégovine",
"Bosniehreh Gercegovinehreh",
"Bosnien an Herzegowina",
"Bosnien och Hercegovina",
"Bosnien un Herzegowina",
"Bosnien und Herzegowina",
"Bosnien-Hercegovina",
"Bosnien-Herzegowina",
"Bosnii Hersegowiin",
"Bosnii da Gercegovin",
"Bosnii na Herzegovinni",
"Bosnij da Gercegovina",
"Bosnija",
"Bosnija bla Gercegovina",
"Bosnija da Gercegovina",
"Bosnija di Khercegovina",
"Bosnija i Gercegovina",
"Bosnija ir Hercegovina",
"Bosnija no Gercegovina",
"Bosnija un Hercegovina",
"Bosnija uonna Khercegovina",
"Bosnijo e Hercegowina",
"Bosnikondre",
"Bosnio kaj Hercegovino",
"Bosnio-Hercegovino",
"Bosniska a Hercegowina",
"Bosniska-Hercegowinska",
"Bosniya",
"Bosniya Harzagobina",
"Bosniya Hersigoviina",
"Bosniya ham Gertsegovina",
"Bosniya hem Hertegovina",
"Bosniya hem Herțegovina",
"Bosniya u Herzegovina",
"Bosniya va Gersegovina",
"Bosniya və Herseqovina",
"Bosniya və Herzokovina",
"Bosniya we Gersegowina",
"Bosniya û Herzegovîna",
"Bosnië Herzegovina",
"Bosnië en Herzegovina",
"Bosnië en Herzegowina",
"Bosnië-Hercegovina",
"Bosnië-Herzegovina",
"Bosnië-Herzegowina",
"Bosniýa we Gersegowina",
"Bosni–Hercegovina",
"Bosnja dhe Hercegovina",
"Bosnje",
"Bosnya a Hersegowina",
"Bosnya asin Hersegobina",
"Bosnya ngan Hersegovina",
"Bosnän e Härzegovän",
"Bosnía og Hersegóvína",
"Bosnïi na Herzegovînni",
"Bosnėjė ėr Hercuogovėna",
"Bossnije-Haezzejovina",
"Bosznia es Hercegovina",
"Bosznia és Hercegovina",
"Bosznia-Hercegovina",
"Boteniya me Erdegobina",
"Boziniya Hezegovina",
"Bozni-Ɛrizigovini",
"Boznia ne Herzegovina",
"Boznija Herzegovina",
"Boznija u Herzegovina",
"Boßnije-Häzzejovina",
"Bośnia i Hercegowina",
"Bośńa a Hercegowina",
"Bożnija u Ħerżegovina",
"Bożnija Ħerżegovina",
"Busna-Hirsiquwina",
"Bòsnia Erzegovina",
"Bòsnia e Ercegovina",
"Bòsnia e Erzegòvina",
"Bòsnia i Hercegovina",
"Bòsnia-Erçegòvina",
"Bòsnijô ë Hercegòwina",
"Bósnia Ercegovina",
"Bósnia e Herzegovina",
"Bósnia-Herzegóvina",
"Bósníà àti Hẹrjẹgòfínà",
"Bô-xni-a Héc-xê-gô-vi-na",
"Bô-xni-a Héc-xê-gô-vi-na (Bosnia và Herzegovina)",
"IBhosinya ne Hezegovi",
"Mbosini ne Hezegovine",
"Narodna Republika Bosna i Hercegovina",
"Orileede Bosinia ati Etisegofina",
"Orílẹ́ède Bọ̀síníà àti Ẹtisẹgófínà",
"People's Republic of Bosnia and Hercegovina",
"People’s Republic of Bosnia and Hercegovina",
"Po-su-ni-a lau Het-set-ko-vi-na",
"Pongia-Herekomina",
"Posinia mo Hesikovinia",
"Posinia mo Hesikōvinia",
"Pô-sṳ-nì-â lâu Het-set-kô-vì-ná",
"Pōngia-Herekōmina",
"Republic of Bosnia and Herzegovina",
"Republika Bosna i Hercegovina",
"Socialist Republic of Bosnia and Hercegovina",
"Socijalisticka Republika Bosna i Hercegovina",
"Socijalistička Republika Bosna i Hercegovina",
"Vonia ha Hesegovina",
"Vosnia kai Erzegovini",
"albwsnh w alhrsk",
"albwsnt w alhrsk",
"albwsnt walhrsk",
"basaniya baro harjegobhina",
"basaniya o harjegobhina",
"basaniya'o harjegobhina",
"basniya",
"basniya mariyu hirjigovina",
"bo si ni ya",
"bo si ni ya he hei sai ge wei na",
"bo si ni ya he hei shan gong he guo",
"bosani'a ate harazegovina",
"bosani'a ebam harjagobhina",
"bosani'a o harjagobhina",
"bosaniya harjigovina",
"boseunia heleuchegobina",
"boseuniaheleuchegobina",
"bosnia da hertsegovina",
"bosnia do hertsegovina",
"bosniya ane harjhegovina",
"bosniya ani harjegovina",
"bosniya ani harjhagovhina",
"bosniya aura harazegovina",
"bosniya aura harzegovina",
"bosniya mariyu herjegovina",
"bosniya mattu harjegovina",
"bosniya mattu herjegovina",
"bosniya ra harjagobhina",
"bosniya ra harjagobhiniya",
"bosniya va harjagovina",
"bsny w hrzgwyn",
"bwsny hrzgwwyn",
"bwsny w hrzgwyn",
"bwsnyh whrzgwbynh",
"bwsnyyە vە ھېrsېgwvyna",
"bwsnʾ whrtsgwbynʾ",
"i-Bosnia ne-Herzegovina",
"pocuniya ercekovina",
"posniya marrum hersikovina",
"Βοσνία - Ερζεγοβίνη",
"Βοσνία και Ερζεγοβίνη",
"Босмудин болн Херцегудин Орн",
"Босна",
"Босна и Херцеговина",
"Босна-Герцеговина",
"Босни æмæ Герцеговинæ",
"Босни а Герцеговина а",
"Босни ба Герцеговина",
"Босни ба Херцеговина",
"Босни тата Герцеговина",
"Босний да Герцеговина",
"Босниэрэ Герцеговинэрэ",
"Босния",
"Босния бла Герцеговина",
"Босния ва Ҳерсеговина",
"Босния да Герцеговина",
"Босния ди Херцеговина",
"Босния және Герцеговина",
"Босния и Герцеговина",
"Босния но Герцеговина",
"Босния уонна Херцеговина",
"Босния һәм Герцеговина",
"Боснія та Герцоговина",
"Боснія і Герцагавіна",
"Боснія і Герцеговина",
"Боснія і Герцеґовина",
"Боснія і Герцоговина",
"Босьнія і Герцагавіна",
"Բոսնիա և Հերցեգովինա",
"Բոսնիա-Հերցեգովինա",
"באסניע און הערצעגאווינע",
"בוסניה והרצגובינה",
"البوسنة و الهرسك",
"البوسنة والهرسك",
"البوسنه و الهرسك",
"بسنی و هرزگوین",
"بوسنىيە ۋە ھېرسېگوۋىنا",
"بوسنی هرزگووین",
"بوسنی و هرزگوین",
"بوسنیا اور ہرزیگووینا",
"بوسنیا تے ہرزیگووینا",
"بوسنیا و ہرزیگووینا",
"بۆسنیا و ھەرزەگۆڤینا",
"ܒܘܣܢܐ ܘ ܗܪܣܟ",
"ܒܘܣܢܐ ܘܗܪܬܣܓܘܒܝܢܐ",
"ބޮސްނިޔާ އެންޑް ހެރްޒިގޮވީނާ",
"बास्निया",
"बॉस्निया आणि हर्झगोव्हिना",
"बॉस्निया और हर्ज़ेगोविना",
"बोसनिया हर्जिगोविना",
"बोस्निया अणि हर्जेगोविना",
"बोस्निया और हरज़ेगोविना",
"बोस्निया र हर्जगोभिना",
"बोस्निया र हर्जगोभिनिया",
"बोस्निया व हर्जगोविना",
"বসনিয়া ও হার্জেগোভিনা",
"বসনিয়া বারো হার্জেগোভিনা",
"বসনিয়াও হার্জেগোভিনা",
"ਬੋਸਨੀਆ ਅਤੇ ਹਰਜ਼ੇਗੋਵੀਨਾ",
"બોસ્નિયા અને હર્ઝેગોવિના",
"ବୋସନିଆ ଏବଂ ହର୍ଜଗୋଭିନା",
"ବୋସନିଆ ଓ ହର୍ଜଗୋଭିନା",
"பொசுனியா எர்செகோவினா",
"போஸ்னியா மற்றும் ஹெர்ஸிகோவினா",
"బాస్నియా మరియు హీర్జిగోవినా",
"బోస్నియా మరియు హెర్జెగొవీనా",
"ಬೊಸ್ನಿಯ ಮತ್ತು ಹೆರ್ಜೆಗೊವಿನ",
"ಬೋಸ್ನಿಯಾ ಮತ್ತು ಹರ್ಜೆಗೋವಿನಾ",
"ബോസ്നിയ ഹെർസെഗോവിന",
"ബോസ്നിയയും ഹെര്സഗോവിനയും",
"බොස්නියා සහ හර්සගෝවිනා",
"බොස්නියාව සහ හර්සගොවීනාව",
"บอสเนียและเฮอร์เซโกวีนา",
"ประเทศบอสเนียและเฮอร์เซโกวีนา",
"ບັອດສເນຍ ແລະ ເຮີດໂກວິເນຍ",
"ປະເທດບົດສະນີແຮກເຊໂກວີນ",
"བོསྣི་ཡ་དང་ཧརྫོ་གོ་ཝི་ན།",
"བྷོསུ་ནིཡ་དང་ཧར་ཛེ་གྷོ་ཝི་ན།",
"ဘော့စနီးယား နှင့် ဟာဇီဂိုဘီးနား",
"ဘော့စနီးယားနှင့် ဟာဇီဂိုဗီးနားနိုင်ငံ",
"ბოსნია და ჰერცეგოვინა",
"ბოსნია დო ჰერცეგოვინა",
"ቦስኒያ እና ሄርዞጎቪኒያ",
"ቦስኒያና ሄርጸጎቪና",
"ᏆᏍᏂᏯ ᎠᎴ ᎲᏤᎪᏫᎾ",
"បូស្ន៉ី",
"ボスニア・ヘルツェゴビナ",
"ボスニア・ヘルツェゴビナ共和国",
"波斯尼亚",
"波斯尼亚和黑塞哥维那",
"波斯尼亚和黑山共和国",
"波斯尼亞",
"보스니아 헤르체고비나",
"보스니아헤르체고비나"
}, target[3].AlternateNames);
#endregion
//Timezone should NOT have underscores
Assert.AreEqual("America/Argentina/Buenos Aires", target[4].Timezone);
//Other misc checks
Assert.AreEqual(new DateTime(2009, 6, 28), target[0].ModificationDate); //DateTime parsing
Assert.AreEqual(3812366000, target[5].Population); //Large (as in: > int.MaxValue) population
Assert.AreEqual(8848, target[6].Elevation); //Elevation
}
private static GeoFile[] GetDumps(GeoFileDownloader downloader)
{
return(new[] {
new GeoFile {
Filename = "admin1CodesASCII.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadAdmin1Codes(fn).Count(); })
},
new GeoFile {
Filename = "admin2Codes.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadAdmin2Codes(fn).Count(); })
},
new GeoFile {
Filename = "allCountries.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "alternateNames.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadAlternateNames(fn).Count(); })
},
new GeoFile {
Filename = "cities1000.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "cities15000.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "cities5000.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "countryInfo.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadCountryInfo(fn).Count(); })
},
//Featurecodes are downloaded by GetCountryDumps()
new GeoFile {
Filename = "hierarchy.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadHierarchy(fn).Count(); })
},
new GeoFile {
Filename = "iso-languagecodes.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadISOLanguageCodes(fn).Count(); })
},
new GeoFile {
Filename = "no-country.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "timeZones.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadTimeZones(fn).Count(); })
},
new GeoFile {
Filename = "userTags.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadUserTags(fn).Count(); })
},
}.Union(GetCountryDumps(downloader)).ToArray());
}
private static GeoFile[] GetCountryDumps(GeoFileDownloader downloader)
{
var w = new WebClient();
var document = w.DownloadString(downloader.BaseUri);
var countries = new Regex("href=\"([A-Z]{2}.zip)", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase)
.Matches(document)
.Cast <Match>()
.Select(m => new GeoFile {
Filename = m.Groups[1].Value, Test = (f) => ExecuteTest(f, (fn) => { return(GeoFileReader.ReadExtendedGeoNames(fn).Count()); })
});
var featurecodes = new Regex("href=\"(featureCodes_[A-Z]{2}.txt)", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase)
.Matches(document)
.Cast <Match>()
.Select(m => new GeoFile {
Filename = m.Groups[1].Value, Test = (f) => ExecuteTest(f, (fn) => { return(GeoFileReader.ReadFeatureCodes(fn).Count()); })
});
return(countries.Union(featurecodes).OrderBy(m => m.Filename).ToArray());
}
static void Main(string[] args)
{
MRArguments argument = UsageChecker.checkAndBuildArgument(args);
var r = new ReverseGeoCode <ExtendedGeoName>(GeoFileReader.ReadExtendedGeoNames(@argument.cityDbFileInfo.FullName));
Parallel.ForEach(argument.targetDirs, (dir) => {
if (dir.Name.Contains("@"))
{
Console.WriteLine("skipping {0}.already renamed. maybe.. it`s name contains @ character", dir.Name);
return;
}
FileInfo[] files = dir.GetFiles();
if (files != null && files.Length > 0)
{
Dictionary <String, int> cCodeCounter = new Dictionary <String, int>();
Dictionary <String, int> cityCounter = new Dictionary <String, int>();
foreach (FileInfo file in files)
{
try {
IEnumerable <MetadataExtractor.Directory> directories = ImageMetadataReader.ReadMetadata(file.FullName);
GpsDirectory gpsDirectory = directories.OfType <GpsDirectory>().First();
if (gpsDirectory != null && gpsDirectory.GetGeoLocation() != null)
{
double lat = gpsDirectory.GetGeoLocation().Latitude;
double lon = gpsDirectory.GetGeoLocation().Longitude;
ExtendedGeoName result = r.NearestNeighbourSearch(r.CreateFromLatLong(lat, lon), 1).First();
String city = result.Name;
if (cityCounter.TryGetValue(city, out int tvalue))
{
cityCounter[city] = tvalue + 1;
}
else
{
cityCounter.Add(city, 1);
}
String ccode = result.CountryCode;
if (cCodeCounter.TryGetValue(ccode, out int cvalue))
{
cCodeCounter[ccode] = cvalue + 1;
}
else
{
cCodeCounter.Add(ccode, 1);
}
}
} catch (Exception) { }
}
if (cCodeCounter.Count > 0)
{
int total = 0;
String countryName = "";
int maxCountryCount = 0;
foreach (String key in cCodeCounter.Keys)
{
int tempCCount = cCodeCounter[key];
total += tempCCount;
if (tempCCount > maxCountryCount)
{
countryName = key;
}
}
String cityName = "";
List <Tuple <String, int> > cities = new List <Tuple <String, int> >();
foreach (String key in cityCounter.Keys)
{
cities.Add(new Tuple <string, int>(key, cityCounter[key]));
}
Tuple <String, int>[] citiesArray = cities.OrderBy(tup => tup.Item2).ToArray();
String destination = dir.Name + " @" + countryName;
if (citiesArray != null && citiesArray.Length > 0)
{
destination += (", " + citiesArray[0].Item1);
cityName = citiesArray[0].Item1;
if (citiesArray.Length > 1)
{
Console.WriteLine("second candidate citi name : {0}, count:{1}", citiesArray[1].Item1, citiesArray[1].Item2);
int c1val = citiesArray[0].Item2;
int c2val = citiesArray[1].Item2;
if (c2val / (float)c1val > 0.4)
{
destination += ("," + citiesArray[1].Item1 + "... ");
}
}
}
bool moved = false;
try {
System.IO.Directory.Move(dir.FullName, dir.Parent.FullName + Path.DirectorySeparatorChar + destination);
moved = true;
} catch (Exception) { }
Console.WriteLine("{0} has {2}/{1} gps data and almost file located [{3}@{4}]. then renaming it to {5} : {6}", dir.Name, files.Length, total, cityName, countryName, destination, moved ? "success" : "failed");
}
else
{
Console.WriteLine("{0}`s {1} file has no gps data", dir.Name, files.Length);
}
}
});
}
