void FileCleanup(GeoFileSet fs)
{
foreach (string f in fs.AllFiles)
{
if (File.Exists(f))
{
_logger.DebugFormat("FileCleanup: {0}", new FileInfo(f).Name);
File.Delete(f);
}
}
}
bool ConvertZipToDat(GeoFileSet fs)
{
bool success = false;
var f = new FileInfo(fs.AllCountriesFile);
//downloaded, now convert zip into serialized dat file
if (File.Exists(fs.AllCountriesFile))
{
if (File.Exists(fs.CountriesRawPath))
{
_logger.DebugFormat("ConvertZipToDat: removing {0}", new FileInfo(fs.CountriesRawPath).Name);
File.Delete(fs.CountriesRawPath);
}
Unzip(fs.AllCountriesFile, Root);
var zd = new DirectoryInfo(Path.Combine(Root, POSTAL_CODE));
if (zd.Exists)
{
zd.Delete(true);
}
zd.Create();
Unzip(fs.AllCountriesPostal, zd.FullName);
if (File.Exists(fs.CountriesRawPath))
{
GeoData gd = ParseGeoFiles(fs);
_logger.DebugFormat("ConvertZipToDat: storing dat => {0}", DataFile);
Serialize.SerializeBinaryToDisk(gd, DataFile);
_logger.Info("ConvertZipToDat: completed");
success = true;
}
else
{
_logger.WarnFormat("ConvertZipToDat: Extraction failed => {0}", f.Name);
}
}
else
{
_logger.WarnFormat("ConvertZipToDat: FileNotFound => {0}", f.Name);
}
return(success);
}
GeoData ParseGeoFiles(GeoFileSet fs)
{
DateTime extractionStart = DateTime.UtcNow;
_logger.Debug("ParseGeoFiles: Begin Extraction");
var gd = new GeoData();
gd.TimeZones = new TimeZoneParser(fs.TimeZonesFile).ParseFile();
gd.FeatureCodes = new FeatureCodeParser(fs.FeatureCodes_EnFile).ParseFile();
gd.Countries = new CountryParser(fs.CountryInfoFile).ParseFile();
gd.GeoNames = new GeoNameParser(fs.CountriesRawPath).ParseFile();
var zf = new FileInfo(fs.PostalsRawPath);
if (zf.Exists)
{
var incCountries = new[] { "US", "CA", "AT", "MX", "GB" };
gd.PostalCodes = new PostalCodeParser(zf.FullName, incCountries).ParseFile();
}
_logger.InfoFormat("ParseGeoFiles: Completed Extraction {0}", DateTime.UtcNow - extractionStart);
return(gd);
}
/// <summary>
/// load file: GeoData - LastModified.txt
/// if available then
/// try to HTTP head the data source <see cref="http://download.geonames.org/export/dump/allCountries.zip"/>
/// pull off last - modified header
/// if newer than the file last-modified, do a GET to download
/// else do a get to download
/// </summary>
internal Task Update(UpdateStep steps)
{
if (steps == UpdateStep.None)
{
throw new InvalidOperationException("steps == UpdateStep.None");
}
return(Task.Factory.StartNew(() =>
{
lock (_lock)
{
var fs = new GeoFileSet(Root)
{
AllCountriesFile = Path.Combine(Root, DATA_FILE + ".zip"),
CountryInfoFile = Path.Combine(Root, "countryInfo.txt"),
FeatureCodes_EnFile = Path.Combine(Root, "featureCodes_en.txt"),
TimeZonesFile = Path.Combine(Root, "timeZones.txt"),
AllCountriesPostal = Path.Combine(Root, POSTAL_CODE + ".zip"),
};
string lastModifiedFile = Path.Combine(Root, LAST_MODIFIED_FILE);
string tmpFile = Path.Combine(Root, Guid.NewGuid().ToString());
bool canReadWrite = steps.HasFlag(UpdateStep.WriteCheck) ? CanWriteTest(tmpFile) : true;
bool shouldDownload = steps.HasFlag(UpdateStep.UpdateCheck) ?
ShouldDownloadCheck(canReadWrite, lastModifiedFile) : true;
if (canReadWrite && shouldDownload)
{
var downloadTasks = new List <Task>();
if (steps.HasFlag(UpdateStep.Download))
{
downloadTasks.Add(DownloadFile(COUNTRY_INFO_URL, fs.CountryInfoFile));
downloadTasks.Add(DownloadFile(FEATURE_CODES_EN_URL, fs.FeatureCodes_EnFile));
downloadTasks.Add(DownloadFile(TIME_ZONE_URL, fs.TimeZonesFile));
downloadTasks.Add(DownloadFile(POSTAL_CODE_URL, fs.AllCountriesPostal));
downloadTasks.Add(DownloadFile(ALL_COUNTRIES_URL, fs.AllCountriesFile, c =>
{
try
{
if (steps.HasFlag(UpdateStep.ChangeModifiedDate))
{
var headers = c.ResponseHeaders;
foreach (string h in headers.Keys)
{
if (h.ToLowerInvariant() == "last-modified")
{
var header = headers[h];
DateTime headerDate = DateTime.Now;
DateTime.TryParse(header.ToString(), out headerDate);
File.WriteAllText(lastModifiedFile, headerDate.ToString());
break;
}
}
}
}
catch (Exception dex)
{
string s = string.Format("DownloadFile: CallBack => {0}", new FileInfo(fs.AllCountriesFile).Name);
_logger.Error(s, dex);
}
}));
Task.WaitAll(downloadTasks.ToArray());
}
if (steps.HasFlag(UpdateStep.Extraction))
{
if (ConvertZipToDat(fs))
{
if (steps.HasFlag(UpdateStep.Cleanup))
{
FileCleanup(fs);
}
else
{
_logger.Warn("Update: No Cleanup flags");
}
}
else
{
_logger.Warn("Update: Dat conversion failed");
}
}
else
{
_logger.Warn("Update: No Extraction flags");
}
}
else
{
_logger.WarnFormat("Update: Not running: canReadWrite=={0} && shouldDownload=={1}", canReadWrite, shouldDownload);
}
}
}));
}
void FileCleanup(GeoFileSet fs)
{
foreach(string f in fs.AllFiles)
{
if (File.Exists(f))
{
_logger.DebugFormat("FileCleanup: {0}", new FileInfo(f).Name);
File.Delete(f);
}
}
}
GeoData ParseGeoFiles(GeoFileSet fs)
{
DateTime extractionStart = DateTime.UtcNow;
_logger.Debug("ParseGeoFiles: Begin Extraction");
var gd = new GeoData();
gd.TimeZones = new TimeZoneParser(fs.TimeZonesFile).ParseFile();
gd.FeatureCodes = new FeatureCodeParser(fs.FeatureCodes_EnFile).ParseFile();
gd.Countries = new CountryParser(fs.CountryInfoFile).ParseFile();
gd.GeoNames = new GeoNameParser(fs.CountriesRawPath).ParseFile();
var zf = new FileInfo(fs.PostalsRawPath);
if (zf.Exists)
{
var incCountries = new[] { "US", "CA", "AT", "MX", "GB" };
gd.PostalCodes = new PostalCodeParser(zf.FullName, incCountries).ParseFile();
}
_logger.InfoFormat("ParseGeoFiles: Completed Extraction {0}", DateTime.UtcNow - extractionStart);
return gd;
}
bool ConvertZipToDat(GeoFileSet fs)
{
bool success = false;
var f = new FileInfo(fs.AllCountriesFile);
//downloaded, now convert zip into serialized dat file
if (File.Exists(fs.AllCountriesFile))
{
if (File.Exists(fs.CountriesRawPath))
{
_logger.DebugFormat("ConvertZipToDat: removing {0}", new FileInfo(fs.CountriesRawPath).Name);
File.Delete(fs.CountriesRawPath);
}
Unzip(fs.AllCountriesFile, Root);
var zd = new DirectoryInfo(Path.Combine(Root, POSTAL_CODE));
if (zd.Exists)
zd.Delete(true);
zd.Create();
Unzip(fs.AllCountriesPostal, zd.FullName);
if (File.Exists(fs.CountriesRawPath))
{
GeoData gd = ParseGeoFiles(fs);
_logger.DebugFormat("ConvertZipToDat: storing dat => {0}", DataFile);
Serialize.SerializeBinaryToDisk(gd, DataFile);
_logger.Info("ConvertZipToDat: completed");
success = true;
}
else
_logger.WarnFormat("ConvertZipToDat: Extraction failed => {0}", f.Name);
}
else
_logger.WarnFormat("ConvertZipToDat: FileNotFound => {0}", f.Name);
return success;
}
/// <summary>
/// load file: GeoData - LastModified.txt
/// if available then
/// try to HTTP head the data source <see cref="http://download.geonames.org/export/dump/allCountries.zip"/>
/// pull off last - modified header
/// if newer than the file last-modified, do a GET to download
/// else do a get to download
/// </summary>
internal Task Update(UpdateStep steps)
{
if (steps == UpdateStep.None)
throw new InvalidOperationException("steps == UpdateStep.None");
return Task.Factory.StartNew(() =>
{
lock (_lock)
{
var fs = new GeoFileSet(Root)
{
AllCountriesFile = Path.Combine(Root, DATA_FILE + ".zip"),
CountryInfoFile = Path.Combine(Root, "countryInfo.txt"),
FeatureCodes_EnFile = Path.Combine(Root, "featureCodes_en.txt"),
TimeZonesFile = Path.Combine(Root, "timeZones.txt"),
AllCountriesPostal = Path.Combine(Root, POSTAL_CODE + ".zip"),
};
string lastModifiedFile = Path.Combine(Root, LAST_MODIFIED_FILE);
string tmpFile = Path.Combine(Root, Guid.NewGuid().ToString());
bool canReadWrite = steps.HasFlag(UpdateStep.WriteCheck) ? CanWriteTest(tmpFile) : true;
bool shouldDownload = steps.HasFlag(UpdateStep.UpdateCheck) ?
ShouldDownloadCheck(canReadWrite, lastModifiedFile) : true;
if (canReadWrite && shouldDownload)
{
var downloadTasks = new List<Task>();
if (steps.HasFlag(UpdateStep.Download))
{
downloadTasks.Add(DownloadFile(COUNTRY_INFO_URL, fs.CountryInfoFile));
downloadTasks.Add(DownloadFile(FEATURE_CODES_EN_URL, fs.FeatureCodes_EnFile));
downloadTasks.Add(DownloadFile(TIME_ZONE_URL, fs.TimeZonesFile));
downloadTasks.Add(DownloadFile(POSTAL_CODE_URL, fs.AllCountriesPostal));
downloadTasks.Add(DownloadFile(ALL_COUNTRIES_URL, fs.AllCountriesFile, c =>
{
try
{
if (steps.HasFlag(UpdateStep.ChangeModifiedDate))
{
var headers = c.ResponseHeaders;
foreach (string h in headers.Keys)
{
if (h.ToLowerInvariant() == "last-modified")
{
var header = headers[h];
DateTime headerDate = DateTime.Now;
DateTime.TryParse(header.ToString(), out headerDate);
File.WriteAllText(lastModifiedFile, headerDate.ToString());
break;
}
}
}
}
catch (Exception dex)
{
string s = string.Format("DownloadFile: CallBack => {0}", new FileInfo(fs.AllCountriesFile).Name);
_logger.Error(s, dex);
}
}));
Task.WaitAll(downloadTasks.ToArray());
}
if (steps.HasFlag(UpdateStep.Extraction))
{
if (ConvertZipToDat(fs))
{
if (steps.HasFlag(UpdateStep.Cleanup))
FileCleanup(fs);
else
_logger.Warn("Update: No Cleanup flags");
}
else
_logger.Warn("Update: Dat conversion failed");
}
else
_logger.Warn("Update: No Extraction flags");
}
else
_logger.WarnFormat("Update: Not running: canReadWrite=={0} && shouldDownload=={1}", canReadWrite, shouldDownload);
}
});
}
