public void Admin1CodesParser_ParsesFileCorrectly()
{
var target = GeoFileReader.ReadAdmin1Codes(@"testdata\test_admin1CodesASCII.txt").ToArray();
Assert.AreEqual(4, target.Length);
Assert.AreEqual("CF.04", target[0].Code);
Assert.AreEqual("Mambéré-Kadéï", target[0].Name);
Assert.AreEqual("Mambere-Kadei", target[0].NameASCII);
Assert.AreEqual(2386161, target[0].GeoNameId);
Assert.AreEqual("This.is.a.VERY.LONG.ID", target[1].Code);
Assert.AreEqual("Iğdır", target[1].Name);
Assert.AreEqual("Igdir", target[1].NameASCII);
Assert.AreEqual(0, target[1].GeoNameId);
Assert.AreEqual("TR.80", target[2].Code);
Assert.AreEqual("Şırnak", target[2].Name);
Assert.AreEqual("Sirnak", target[2].NameASCII);
Assert.AreEqual(443189, target[2].GeoNameId);
Assert.AreEqual("UA.26", target[3].Code);
Assert.AreEqual("Zaporiz’ka Oblast’", target[3].Name);
Assert.AreEqual("Zaporiz'ka Oblast'", target[3].NameASCII);
Assert.AreEqual(687699, target[3].GeoNameId);
}
void EnsureLoaded()
{
if (this.geodata == null)
{
lock (this.syncLock)
{
if (this.geodata == null)
{
using (var stream = this.LoadStream("admin1CodesASCII.txt"))
{
this.adminCodes = GeoFileReader
.ReadAdmin1Codes(stream)
.ToList();
}
using (var stream = this.LoadStream("cities5000.txt"))
{
this.geodata = GeoFileReader
.ReadExtendedGeoNames(stream)
.Where(x =>
//x.FeatureCode.Equals("PPL", StringComparison.OrdinalIgnoreCase) &&
//(
x.CountryCode.Equals("CA", StringComparison.CurrentCultureIgnoreCase) ||
x.CountryCode.Equals("US", StringComparison.CurrentCultureIgnoreCase)
//)
)
.OrderBy(x => x.Latitude)
.ThenBy(x => x.Longitude)
.ToList();
}
}
}
}
}
public void Admin1CodesComposer_ComposesFileCorrectly()
{
var src = @"testdata\test_admin1CodesASCII.txt";
var dst = @"testdata\test_admin1CodesASCII.out.txt";
GeoFileWriter.WriteAdmin1Codes(dst, GeoFileReader.ReadAdmin1Codes(src));
FileUtil.EnsureFilesAreFunctionallyEqual(src, dst, 4, 0, new[] { '\t' }, Encoding.UTF8, false);
}
/// <summary>
/// Getting GeoData
/// </summary>
/// <param name="appSettings">to know where to store the temp files</param>
/// <param name="geoFileDownload">Abstraction to download Geo Data</param>
/// <param name="memoryCache">for keeping status</param>
public GeoReverseLookup(AppSettings appSettings, IGeoFileDownload geoFileDownload, IMemoryCache memoryCache = null, IWebLogger logger = null)
{
// Needed when not having this, application will fail
geoFileDownload.Download().ConfigureAwait(false);
_appSettings = appSettings;
_logger = logger;
_admin1CodesAscii = GeoFileReader.ReadAdmin1Codes(
Path.Combine(appSettings.TempFolder, "admin1CodesASCII.txt"));
// Create our ReverseGeoCode class and supply it with data
InitReverseGeoCode();
_cache = memoryCache;
}
private static void DumpASCIILies(string logpath)
{
using (var lw = File.CreateText(Path.Combine(logpath, "_asciilies.log")))
{
//Test for fields that claim to contain ASCII only but contain non-ASCII data anyways
var nonasciifilter = new Regex("[^\x20-\x7F]", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
var geofilefilter = new Regex("^[A-Z]{2}.txt$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
lw.WriteLine("The following files contain entries that claim to contain ASCII only but contain non-ASCII data anyways:");
var extgeofiles = new[] { "allCountries", "cities1000", "cities5000", "cities15000", "no-country" }
.Select(f => Path.Combine(Dump_DownloadDirectory, f + ".txt"))
.Union(Directory.GetFiles(Dump_DownloadDirectory, "*.txt")
.Where(f => geofilefilter.IsMatch(Path.GetFileName(f)))
);
var lies = extgeofiles.AsParallel()
.SelectMany(f => GeoFileReader.ReadExtendedGeoNames(f)
.Where(e => nonasciifilter.IsMatch(e.NameASCII))
.Select(i => new NonASCIIEntry {
FileName = f, Id = i.Id, Value = i.NameASCII
})
).Union(
GeoFileReader.ReadAdmin1Codes(Path.Combine(Dump_DownloadDirectory, "admin1CodesASCII.txt")).AsParallel()
.Where(c => nonasciifilter.IsMatch(c.NameASCII))
.Select(i => new NonASCIIEntry {
FileName = "admin1CodesASCII.txt", Id = i.GeoNameId, Value = i.NameASCII
})
).Union(
GeoFileReader.ReadAdmin2Codes(Path.Combine(Dump_DownloadDirectory, "admin2Codes.txt")).AsParallel()
.Where(c => nonasciifilter.IsMatch(c.NameASCII))
.Select(i => new NonASCIIEntry {
FileName = "admin2Codes.txt", Id = i.GeoNameId, Value = i.NameASCII
})
);
foreach (var l in lies.OrderBy(l => l.FileName).ThenBy(l => l.Value))
{
lw.WriteLine(string.Join("\t", Path.GetFileName(l.FileName), l.Id, l.Value));
}
;
}
}
public void FileReader_Admin1Codes_StreamOverload()
{
using (var s = File.OpenRead(@"testdata\test_admin1CodesASCII.txt"))
GeoFileReader.ReadAdmin1Codes(s).Count();
}
public void ParserThrowsOnNonExistingFile()
{
var target = GeoFileReader.ReadAdmin1Codes(@"testdata\non_existing_file.txt").ToArray();
}
public void ParserThrowsOnInvalidData()
{
var target = GeoFileReader.ReadAdmin1Codes(@"testdata\invalid_admin1CodesASCII.txt").ToArray();
}
private static GeoFile[] GetDumps(GeoFileDownloader downloader)
{
return(new[] {
new GeoFile {
Filename = "admin1CodesASCII.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadAdmin1Codes(fn).Count(); })
},
new GeoFile {
Filename = "admin2Codes.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadAdmin2Codes(fn).Count(); })
},
new GeoFile {
Filename = "allCountries.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "alternateNames.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadAlternateNames(fn).Count(); })
},
new GeoFile {
Filename = "cities1000.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "cities15000.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "cities5000.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "countryInfo.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadCountryInfo(fn).Count(); })
},
//Featurecodes are downloaded by GetCountryDumps()
new GeoFile {
Filename = "hierarchy.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadHierarchy(fn).Count(); })
},
new GeoFile {
Filename = "iso-languagecodes.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadISOLanguageCodes(fn).Count(); })
},
new GeoFile {
Filename = "no-country.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadExtendedGeoNames(fn).Count(); })
},
new GeoFile {
Filename = "timeZones.txt", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadTimeZones(fn).Count(); })
},
new GeoFile {
Filename = "userTags.zip", Test = (f) => ExecuteTest(f, (fn) => { return GeoFileReader.ReadUserTags(fn).Count(); })
},
}.Union(GetCountryDumps(downloader)).ToArray());
}
