void FileCleanup(GeoFileSet fs) { foreach (string f in fs.AllFiles) { if (File.Exists(f)) { _logger.DebugFormat("FileCleanup: {0}", new FileInfo(f).Name); File.Delete(f); } } }
bool ConvertZipToDat(GeoFileSet fs) { bool success = false; var f = new FileInfo(fs.AllCountriesFile); //downloaded, now convert zip into serialized dat file if (File.Exists(fs.AllCountriesFile)) { if (File.Exists(fs.CountriesRawPath)) { _logger.DebugFormat("ConvertZipToDat: removing {0}", new FileInfo(fs.CountriesRawPath).Name); File.Delete(fs.CountriesRawPath); } Unzip(fs.AllCountriesFile, Root); var zd = new DirectoryInfo(Path.Combine(Root, POSTAL_CODE)); if (zd.Exists) { zd.Delete(true); } zd.Create(); Unzip(fs.AllCountriesPostal, zd.FullName); if (File.Exists(fs.CountriesRawPath)) { GeoData gd = ParseGeoFiles(fs); _logger.DebugFormat("ConvertZipToDat: storing dat => {0}", DataFile); Serialize.SerializeBinaryToDisk(gd, DataFile); _logger.Info("ConvertZipToDat: completed"); success = true; } else { _logger.WarnFormat("ConvertZipToDat: Extraction failed => {0}", f.Name); } } else { _logger.WarnFormat("ConvertZipToDat: FileNotFound => {0}", f.Name); } return(success); }
GeoData ParseGeoFiles(GeoFileSet fs) { DateTime extractionStart = DateTime.UtcNow; _logger.Debug("ParseGeoFiles: Begin Extraction"); var gd = new GeoData(); gd.TimeZones = new TimeZoneParser(fs.TimeZonesFile).ParseFile(); gd.FeatureCodes = new FeatureCodeParser(fs.FeatureCodes_EnFile).ParseFile(); gd.Countries = new CountryParser(fs.CountryInfoFile).ParseFile(); gd.GeoNames = new GeoNameParser(fs.CountriesRawPath).ParseFile(); var zf = new FileInfo(fs.PostalsRawPath); if (zf.Exists) { var incCountries = new[] { "US", "CA", "AT", "MX", "GB" }; gd.PostalCodes = new PostalCodeParser(zf.FullName, incCountries).ParseFile(); } _logger.InfoFormat("ParseGeoFiles: Completed Extraction {0}", DateTime.UtcNow - extractionStart); return(gd); }
/// <summary> /// load file: GeoData - LastModified.txt /// if available then /// try to HTTP head the data source <see cref="http://download.geonames.org/export/dump/allCountries.zip"/> /// pull off last - modified header /// if newer than the file last-modified, do a GET to download /// else do a get to download /// </summary> internal Task Update(UpdateStep steps) { if (steps == UpdateStep.None) { throw new InvalidOperationException("steps == UpdateStep.None"); } return(Task.Factory.StartNew(() => { lock (_lock) { var fs = new GeoFileSet(Root) { AllCountriesFile = Path.Combine(Root, DATA_FILE + ".zip"), CountryInfoFile = Path.Combine(Root, "countryInfo.txt"), FeatureCodes_EnFile = Path.Combine(Root, "featureCodes_en.txt"), TimeZonesFile = Path.Combine(Root, "timeZones.txt"), AllCountriesPostal = Path.Combine(Root, POSTAL_CODE + ".zip"), }; string lastModifiedFile = Path.Combine(Root, LAST_MODIFIED_FILE); string tmpFile = Path.Combine(Root, Guid.NewGuid().ToString()); bool canReadWrite = steps.HasFlag(UpdateStep.WriteCheck) ? CanWriteTest(tmpFile) : true; bool shouldDownload = steps.HasFlag(UpdateStep.UpdateCheck) ? ShouldDownloadCheck(canReadWrite, lastModifiedFile) : true; if (canReadWrite && shouldDownload) { var downloadTasks = new List <Task>(); if (steps.HasFlag(UpdateStep.Download)) { downloadTasks.Add(DownloadFile(COUNTRY_INFO_URL, fs.CountryInfoFile)); downloadTasks.Add(DownloadFile(FEATURE_CODES_EN_URL, fs.FeatureCodes_EnFile)); downloadTasks.Add(DownloadFile(TIME_ZONE_URL, fs.TimeZonesFile)); downloadTasks.Add(DownloadFile(POSTAL_CODE_URL, fs.AllCountriesPostal)); downloadTasks.Add(DownloadFile(ALL_COUNTRIES_URL, fs.AllCountriesFile, c => { try { if (steps.HasFlag(UpdateStep.ChangeModifiedDate)) { var headers = c.ResponseHeaders; foreach (string h in headers.Keys) { if (h.ToLowerInvariant() == "last-modified") { var header = headers[h]; DateTime headerDate = DateTime.Now; DateTime.TryParse(header.ToString(), out headerDate); File.WriteAllText(lastModifiedFile, headerDate.ToString()); break; } } } } catch (Exception dex) { string s = string.Format("DownloadFile: CallBack => {0}", new FileInfo(fs.AllCountriesFile).Name); _logger.Error(s, dex); } })); Task.WaitAll(downloadTasks.ToArray()); } if (steps.HasFlag(UpdateStep.Extraction)) { if (ConvertZipToDat(fs)) { if (steps.HasFlag(UpdateStep.Cleanup)) { FileCleanup(fs); } else { _logger.Warn("Update: No Cleanup flags"); } } else { _logger.Warn("Update: Dat conversion failed"); } } else { _logger.Warn("Update: No Extraction flags"); } } else { _logger.WarnFormat("Update: Not running: canReadWrite=={0} && shouldDownload=={1}", canReadWrite, shouldDownload); } } })); }
void FileCleanup(GeoFileSet fs) { foreach(string f in fs.AllFiles) { if (File.Exists(f)) { _logger.DebugFormat("FileCleanup: {0}", new FileInfo(f).Name); File.Delete(f); } } }
GeoData ParseGeoFiles(GeoFileSet fs) { DateTime extractionStart = DateTime.UtcNow; _logger.Debug("ParseGeoFiles: Begin Extraction"); var gd = new GeoData(); gd.TimeZones = new TimeZoneParser(fs.TimeZonesFile).ParseFile(); gd.FeatureCodes = new FeatureCodeParser(fs.FeatureCodes_EnFile).ParseFile(); gd.Countries = new CountryParser(fs.CountryInfoFile).ParseFile(); gd.GeoNames = new GeoNameParser(fs.CountriesRawPath).ParseFile(); var zf = new FileInfo(fs.PostalsRawPath); if (zf.Exists) { var incCountries = new[] { "US", "CA", "AT", "MX", "GB" }; gd.PostalCodes = new PostalCodeParser(zf.FullName, incCountries).ParseFile(); } _logger.InfoFormat("ParseGeoFiles: Completed Extraction {0}", DateTime.UtcNow - extractionStart); return gd; }
bool ConvertZipToDat(GeoFileSet fs) { bool success = false; var f = new FileInfo(fs.AllCountriesFile); //downloaded, now convert zip into serialized dat file if (File.Exists(fs.AllCountriesFile)) { if (File.Exists(fs.CountriesRawPath)) { _logger.DebugFormat("ConvertZipToDat: removing {0}", new FileInfo(fs.CountriesRawPath).Name); File.Delete(fs.CountriesRawPath); } Unzip(fs.AllCountriesFile, Root); var zd = new DirectoryInfo(Path.Combine(Root, POSTAL_CODE)); if (zd.Exists) zd.Delete(true); zd.Create(); Unzip(fs.AllCountriesPostal, zd.FullName); if (File.Exists(fs.CountriesRawPath)) { GeoData gd = ParseGeoFiles(fs); _logger.DebugFormat("ConvertZipToDat: storing dat => {0}", DataFile); Serialize.SerializeBinaryToDisk(gd, DataFile); _logger.Info("ConvertZipToDat: completed"); success = true; } else _logger.WarnFormat("ConvertZipToDat: Extraction failed => {0}", f.Name); } else _logger.WarnFormat("ConvertZipToDat: FileNotFound => {0}", f.Name); return success; }
/// <summary> /// load file: GeoData - LastModified.txt /// if available then /// try to HTTP head the data source <see cref="http://download.geonames.org/export/dump/allCountries.zip"/> /// pull off last - modified header /// if newer than the file last-modified, do a GET to download /// else do a get to download /// </summary> internal Task Update(UpdateStep steps) { if (steps == UpdateStep.None) throw new InvalidOperationException("steps == UpdateStep.None"); return Task.Factory.StartNew(() => { lock (_lock) { var fs = new GeoFileSet(Root) { AllCountriesFile = Path.Combine(Root, DATA_FILE + ".zip"), CountryInfoFile = Path.Combine(Root, "countryInfo.txt"), FeatureCodes_EnFile = Path.Combine(Root, "featureCodes_en.txt"), TimeZonesFile = Path.Combine(Root, "timeZones.txt"), AllCountriesPostal = Path.Combine(Root, POSTAL_CODE + ".zip"), }; string lastModifiedFile = Path.Combine(Root, LAST_MODIFIED_FILE); string tmpFile = Path.Combine(Root, Guid.NewGuid().ToString()); bool canReadWrite = steps.HasFlag(UpdateStep.WriteCheck) ? CanWriteTest(tmpFile) : true; bool shouldDownload = steps.HasFlag(UpdateStep.UpdateCheck) ? ShouldDownloadCheck(canReadWrite, lastModifiedFile) : true; if (canReadWrite && shouldDownload) { var downloadTasks = new List<Task>(); if (steps.HasFlag(UpdateStep.Download)) { downloadTasks.Add(DownloadFile(COUNTRY_INFO_URL, fs.CountryInfoFile)); downloadTasks.Add(DownloadFile(FEATURE_CODES_EN_URL, fs.FeatureCodes_EnFile)); downloadTasks.Add(DownloadFile(TIME_ZONE_URL, fs.TimeZonesFile)); downloadTasks.Add(DownloadFile(POSTAL_CODE_URL, fs.AllCountriesPostal)); downloadTasks.Add(DownloadFile(ALL_COUNTRIES_URL, fs.AllCountriesFile, c => { try { if (steps.HasFlag(UpdateStep.ChangeModifiedDate)) { var headers = c.ResponseHeaders; foreach (string h in headers.Keys) { if (h.ToLowerInvariant() == "last-modified") { var header = headers[h]; DateTime headerDate = DateTime.Now; DateTime.TryParse(header.ToString(), out headerDate); File.WriteAllText(lastModifiedFile, headerDate.ToString()); break; } } } } catch (Exception dex) { string s = string.Format("DownloadFile: CallBack => {0}", new FileInfo(fs.AllCountriesFile).Name); _logger.Error(s, dex); } })); Task.WaitAll(downloadTasks.ToArray()); } if (steps.HasFlag(UpdateStep.Extraction)) { if (ConvertZipToDat(fs)) { if (steps.HasFlag(UpdateStep.Cleanup)) FileCleanup(fs); else _logger.Warn("Update: No Cleanup flags"); } else _logger.Warn("Update: Dat conversion failed"); } else _logger.Warn("Update: No Extraction flags"); } else _logger.WarnFormat("Update: Not running: canReadWrite=={0} && shouldDownload=={1}", canReadWrite, shouldDownload); } }); }