static Inner()
{
try {
if (System.IO.File.Exists(DataManager.Instance.DataFile))
SINGLETON = Serialize.DeserializeBinaryFromDisk<GeoData>(DataManager.Instance.DataFile);
else
SINGLETON = Serialize.DeserializeBinaryFromResource<GeoData>("GeoDataSource.GeoDataSource.dat");
if (SINGLETON == null)
throw new ApplicationException("Unable to deserialize dat file");
{
Dictionary<string, Country> iso2Map = (from c in SINGLETON.Countries
where c != null && !string.IsNullOrWhiteSpace(c.ISOAlpha2)
group c by c.ISOAlpha2 into cg
select cg).ToDictionary(g => g.Key.ToLower().Trim(), g => g.FirstOrDefault());
SINGLETON.LinkNamesInfos(iso2Map);
SINGLETON.LinkPostalInfos(iso2Map);
}
SINGLETON.BuildPostalHash();
}
catch(Exception ex)
{
_logger.Fatal("Inner static CTOR", ex);
throw;
}
}
public static GeoData ParseFile(string GeoNameFile, string TimeZoneFile, string FeatureCodeFile, string countryInfoFile)
{
var gd = new GeoData();
gd.TimeZones = TimeZoneParser.ParseFile(TimeZoneFile);
gd.FeatureCodes = FeatureCodeParser.ParseFile(FeatureCodeFile);
gd.Countries = CountryParser.ParseFile(countryInfoFile);
List<GeoName> Names = new List<GeoName>();
int count = 0;
using (System.IO.StreamReader rdr = new System.IO.StreamReader(GeoNameFile))
{
string line = "";
do
{
line = rdr.ReadLine();
if (!string.IsNullOrEmpty(line))
{
GeoName n = ParseLine(line);
if (n.FeatureClass.StartsWith("ADM1"))// || n.FeatureClass.StartsWith("ADM2"))
{
Names.Add(n);
}
}
count++;
} while (!string.IsNullOrEmpty(line));
}
gd.GeoNames = Names;
return gd;
}
bool ConvertZipToDat(GeoFileSet fs)
{
bool success = false;
var f = new FileInfo(fs.AllCountriesFile);
//downloaded, now convert zip into serialized dat file
if (File.Exists(fs.AllCountriesFile))
{
if (File.Exists(fs.CountriesRawPath))
{
_logger.DebugFormat("ConvertZipToDat: removing {0}", new FileInfo(fs.CountriesRawPath).Name);
File.Delete(fs.CountriesRawPath);
}
Unzip(fs.AllCountriesFile, Root);
var zd = new DirectoryInfo(Path.Combine(Root, POSTAL_CODE));
if (zd.Exists)
{
zd.Delete(true);
}
zd.Create();
Unzip(fs.AllCountriesPostal, zd.FullName);
if (File.Exists(fs.CountriesRawPath))
{
GeoData gd = ParseGeoFiles(fs);
_logger.DebugFormat("ConvertZipToDat: storing dat => {0}", DataFile);
Serialize.SerializeBinaryToDisk(gd, DataFile);
_logger.Info("ConvertZipToDat: completed");
success = true;
}
else
{
_logger.WarnFormat("ConvertZipToDat: Extraction failed => {0}", f.Name);
}
}
else
{
_logger.WarnFormat("ConvertZipToDat: FileNotFound => {0}", f.Name);
}
return(success);
}
GeoData ParseGeoFiles(GeoFileSet fs)
{
DateTime extractionStart = DateTime.UtcNow;
_logger.Debug("ParseGeoFiles: Begin Extraction");
var gd = new GeoData();
gd.TimeZones = new TimeZoneParser(fs.TimeZonesFile).ParseFile();
gd.FeatureCodes = new FeatureCodeParser(fs.FeatureCodes_EnFile).ParseFile();
gd.Countries = new CountryParser(fs.CountryInfoFile).ParseFile();
gd.GeoNames = new GeoNameParser(fs.CountriesRawPath).ParseFile();
var zf = new FileInfo(fs.PostalsRawPath);
if (zf.Exists)
{
var incCountries = new[] { "US", "CA", "AT", "MX", "GB" };
gd.PostalCodes = new PostalCodeParser(zf.FullName, incCountries).ParseFile();
}
_logger.InfoFormat("ParseGeoFiles: Completed Extraction {0}", DateTime.UtcNow - extractionStart);
return(gd);
}
GeoData ParseGeoFiles(GeoFileSet fs)
{
DateTime extractionStart = DateTime.UtcNow;
_logger.Debug("ParseGeoFiles: Begin Extraction");
var gd = new GeoData();
gd.TimeZones = new TimeZoneParser(fs.TimeZonesFile).ParseFile();
gd.FeatureCodes = new FeatureCodeParser(fs.FeatureCodes_EnFile).ParseFile();
gd.Countries = new CountryParser(fs.CountryInfoFile).ParseFile();
gd.GeoNames = new GeoNameParser(fs.CountriesRawPath).ParseFile();
var zf = new FileInfo(fs.PostalsRawPath);
if (zf.Exists)
{
var incCountries = new[] { "US", "CA", "AT", "MX", "GB" };
gd.PostalCodes = new PostalCodeParser(zf.FullName, incCountries).ParseFile();
}
_logger.InfoFormat("ParseGeoFiles: Completed Extraction {0}", DateTime.UtcNow - extractionStart);
return gd;
}
public static Task Update()
{
return(Task.Factory.StartNew(() =>
{
lock (_lock)
{
string lastModifiedFile = System.IO.Path.Combine(Root, LastModified);
string tmpFile = System.IO.Path.Combine(Root, Guid.NewGuid().ToString());
countriesRawFile = System.IO.Path.Combine(Root, countriesRawFile);
string allCountriesFile = System.IO.Path.Combine(Root, dataFile + ".zip");
string countryInfoFile = System.IO.Path.Combine(Root, "countryInfo.txt");
string featureCodes_enFile = System.IO.Path.Combine(Root, "featureCodes_en.txt");
string timeZonesFile = System.IO.Path.Combine(Root, "timeZones.txt");
//check to see if we have read/write persmission to disk
bool canReadWrite = false;
try
{
System.IO.File.WriteAllText(tmpFile, "testing write access");
string contents = System.IO.File.ReadAllText(tmpFile);
canReadWrite = !string.IsNullOrEmpty(contents);
System.IO.File.Delete(tmpFile);
}
catch (Exception)
{
}
//load file: GeoData-LastModified.txt
//if available then
//try to http head the data source url: http://download.geonames.org/export/dump/allCountries.zip
//pull off last-modified header
//if newer than the file last-modified, do a GET to download
//else
// do a get to download
bool shouldDownload = !System.IO.File.Exists(lastModifiedFile);
if (canReadWrite && System.IO.File.Exists(lastModifiedFile))
{
var lastModified = System.IO.File.ReadAllText(lastModifiedFile);
DateTime lastModifiedDate;
if (!DateTime.TryParse(lastModified, out lastModifiedDate))
{
shouldDownload = true;
}
else
{
var request = new WebClientRequest();
request.Headers = new WebHeaderCollection();
var client = new ParallelWebClient(request);
client.OpenReadTask(allCountriesUrl, "HEAD").ContinueWith(t =>
{
var headers = client.ResponseHeaders;
foreach (string h in headers.Keys)
{
if (h.ToLowerInvariant() == "last-modified")
{
var header = headers[h];
DateTime headerDate = DateTime.Now;
DateTime.TryParse(header.ToString(), out headerDate);
if (lastModifiedDate < headerDate)
{
shouldDownload = true;
}
break;
}
}
}).Wait();
}
}
if (canReadWrite && shouldDownload)
{
var downloadTasks = new List <Task>();
var request = new WebClientRequest();
request.Headers = new WebHeaderCollection();
var client = new ParallelWebClient(request);
downloadTasks.Add(
client.DownloadData(allCountriesUrl).ContinueWith(t =>
{
if (System.IO.File.Exists(allCountriesFile))
{
System.IO.File.Delete(allCountriesFile);
}
System.IO.File.WriteAllBytes(allCountriesFile, t.Result);
var headers = client.ResponseHeaders;
foreach (string h in headers.Keys)
{
if (h.ToLowerInvariant() == "last-modified")
{
var header = headers[h];
DateTime headerDate = DateTime.Now;
DateTime.TryParse(header.ToString(), out headerDate);
System.IO.File.WriteAllText(lastModifiedFile, headerDate.ToString());
break;
}
}
})
);
downloadTasks.Add(client.DownloadData(countryInfoUrl).ContinueWith(t =>
{
if (System.IO.File.Exists(countryInfoFile))
{
System.IO.File.Delete(countryInfoFile);
}
System.IO.File.WriteAllBytes(countryInfoFile, t.Result);
}));
downloadTasks.Add(client.DownloadData(featureCodes_enUrl).ContinueWith(t =>
{
if (System.IO.File.Exists(featureCodes_enFile))
{
System.IO.File.Delete(featureCodes_enFile);
}
System.IO.File.WriteAllBytes(featureCodes_enFile, t.Result);
}));
downloadTasks.Add(client.DownloadData(timeZonesUrl).ContinueWith(t =>
{
if (System.IO.File.Exists(timeZonesFile))
{
System.IO.File.Delete(timeZonesFile);
}
System.IO.File.WriteAllBytes(timeZonesFile, t.Result);
}));
Task.WaitAll(downloadTasks.ToArray());
//downloaded, now convert zip into serialized dat file
if (System.IO.File.Exists(allCountriesFile))
{
if (System.IO.File.Exists(countriesRawFile))
{
System.IO.File.Delete(countriesRawFile);
}
ICSharpCode.SharpZipLib.Zip.FastZip fz = new FastZip();
fz.ExtractZip(allCountriesFile, Root, FastZip.Overwrite.Always, null, null, null, true);
if (System.IO.File.Exists(countriesRawFile))
{
GeoData gd = GeoNameParser.ParseFile(countriesRawFile, timeZonesFile,
featureCodes_enFile, countryInfoFile);
Serialize.SerializeBinaryToDisk(gd, DataFile);
}
if (System.IO.File.Exists(allCountriesFile))
{
System.IO.File.Delete(allCountriesFile);
}
if (System.IO.File.Exists(countriesRawFile))
{
System.IO.File.Delete(countriesRawFile);
}
if (System.IO.File.Exists(countryInfoFile))
{
System.IO.File.Delete(countryInfoFile);
}
if (System.IO.File.Exists(featureCodes_enFile))
{
System.IO.File.Delete(featureCodes_enFile);
}
if (System.IO.File.Exists(timeZonesFile))
{
System.IO.File.Delete(timeZonesFile);
}
}
}
}
}));
}
