Exemple #1
0
            static Inner()
            {
                try {
                    if (System.IO.File.Exists(DataManager.Instance.DataFile))
                        SINGLETON = Serialize.DeserializeBinaryFromDisk<GeoData>(DataManager.Instance.DataFile);
                    else
                        SINGLETON = Serialize.DeserializeBinaryFromResource<GeoData>("GeoDataSource.GeoDataSource.dat");

                    if (SINGLETON == null)
                        throw new ApplicationException("Unable to deserialize dat file");

                    {
                        Dictionary<string, Country> iso2Map = (from c in SINGLETON.Countries
                                                               where c != null && !string.IsNullOrWhiteSpace(c.ISOAlpha2)
                                                               group c by c.ISOAlpha2 into cg
                                                               select cg).ToDictionary(g => g.Key.ToLower().Trim(), g => g.FirstOrDefault());
                        SINGLETON.LinkNamesInfos(iso2Map);
                        SINGLETON.LinkPostalInfos(iso2Map);
                    }
                    SINGLETON.BuildPostalHash();
                }
                catch(Exception ex)
                {
                    _logger.Fatal("Inner static CTOR", ex);
                    throw;
                }
            }
        public static GeoData ParseFile(string GeoNameFile, string TimeZoneFile, string FeatureCodeFile, string countryInfoFile)
        {
            var gd = new GeoData();
            gd.TimeZones  = TimeZoneParser.ParseFile(TimeZoneFile);
            gd.FeatureCodes = FeatureCodeParser.ParseFile(FeatureCodeFile);
            gd.Countries = CountryParser.ParseFile(countryInfoFile);

            List<GeoName> Names = new List<GeoName>();
            int count = 0;
            using (System.IO.StreamReader rdr = new System.IO.StreamReader(GeoNameFile))
            {
                string line = "";
                do
                {
                    line = rdr.ReadLine();
                    if (!string.IsNullOrEmpty(line))
                    {
                        GeoName n = ParseLine(line);
                        if (n.FeatureClass.StartsWith("ADM1"))// || n.FeatureClass.StartsWith("ADM2"))
                        {
                            Names.Add(n);
                        }

                    }
                    count++;
                } while (!string.IsNullOrEmpty(line));
            }
            gd.GeoNames = Names;
            return gd;
        }
Exemple #3
0
        bool ConvertZipToDat(GeoFileSet fs)
        {
            bool success = false;
            var  f       = new FileInfo(fs.AllCountriesFile);

            //downloaded, now convert zip into serialized dat file
            if (File.Exists(fs.AllCountriesFile))
            {
                if (File.Exists(fs.CountriesRawPath))
                {
                    _logger.DebugFormat("ConvertZipToDat: removing {0}", new FileInfo(fs.CountriesRawPath).Name);
                    File.Delete(fs.CountriesRawPath);
                }

                Unzip(fs.AllCountriesFile, Root);

                var zd = new DirectoryInfo(Path.Combine(Root, POSTAL_CODE));
                if (zd.Exists)
                {
                    zd.Delete(true);
                }

                zd.Create();
                Unzip(fs.AllCountriesPostal, zd.FullName);

                if (File.Exists(fs.CountriesRawPath))
                {
                    GeoData gd = ParseGeoFiles(fs);

                    _logger.DebugFormat("ConvertZipToDat: storing dat => {0}", DataFile);
                    Serialize.SerializeBinaryToDisk(gd, DataFile);
                    _logger.Info("ConvertZipToDat: completed");
                    success = true;
                }
                else
                {
                    _logger.WarnFormat("ConvertZipToDat: Extraction failed => {0}", f.Name);
                }
            }
            else
            {
                _logger.WarnFormat("ConvertZipToDat: FileNotFound => {0}", f.Name);
            }

            return(success);
        }
Exemple #4
0
        GeoData ParseGeoFiles(GeoFileSet fs)
        {
            DateTime extractionStart = DateTime.UtcNow;

            _logger.Debug("ParseGeoFiles: Begin Extraction");

            var gd = new GeoData();

            gd.TimeZones    = new TimeZoneParser(fs.TimeZonesFile).ParseFile();
            gd.FeatureCodes = new FeatureCodeParser(fs.FeatureCodes_EnFile).ParseFile();
            gd.Countries    = new CountryParser(fs.CountryInfoFile).ParseFile();
            gd.GeoNames     = new GeoNameParser(fs.CountriesRawPath).ParseFile();

            var zf = new FileInfo(fs.PostalsRawPath);

            if (zf.Exists)
            {
                var incCountries = new[] { "US", "CA", "AT", "MX", "GB" };
                gd.PostalCodes = new PostalCodeParser(zf.FullName, incCountries).ParseFile();
            }
            _logger.InfoFormat("ParseGeoFiles: Completed Extraction {0}", DateTime.UtcNow - extractionStart);
            return(gd);
        }
Exemple #5
0
        GeoData ParseGeoFiles(GeoFileSet fs)
        {
            DateTime extractionStart = DateTime.UtcNow;
            _logger.Debug("ParseGeoFiles: Begin Extraction");

            var gd = new GeoData();            
            gd.TimeZones = new TimeZoneParser(fs.TimeZonesFile).ParseFile();
            gd.FeatureCodes = new FeatureCodeParser(fs.FeatureCodes_EnFile).ParseFile();
            gd.Countries = new CountryParser(fs.CountryInfoFile).ParseFile();
            gd.GeoNames = new GeoNameParser(fs.CountriesRawPath).ParseFile();

            var zf = new FileInfo(fs.PostalsRawPath);
            if (zf.Exists)
            {
                var incCountries = new[] { "US", "CA", "AT", "MX", "GB" };
                gd.PostalCodes = new PostalCodeParser(zf.FullName, incCountries).ParseFile();
            }
            _logger.InfoFormat("ParseGeoFiles: Completed Extraction {0}", DateTime.UtcNow - extractionStart);
            return gd;
        }
Exemple #6
0
        public static Task Update()
        {
            return(Task.Factory.StartNew(() =>
            {
                lock (_lock)
                {
                    string lastModifiedFile = System.IO.Path.Combine(Root, LastModified);
                    string tmpFile = System.IO.Path.Combine(Root, Guid.NewGuid().ToString());

                    countriesRawFile = System.IO.Path.Combine(Root, countriesRawFile);
                    string allCountriesFile = System.IO.Path.Combine(Root, dataFile + ".zip");
                    string countryInfoFile = System.IO.Path.Combine(Root, "countryInfo.txt");
                    string featureCodes_enFile = System.IO.Path.Combine(Root, "featureCodes_en.txt");
                    string timeZonesFile = System.IO.Path.Combine(Root, "timeZones.txt");


                    //check to see if we have read/write persmission to disk
                    bool canReadWrite = false;
                    try
                    {
                        System.IO.File.WriteAllText(tmpFile, "testing write access");
                        string contents = System.IO.File.ReadAllText(tmpFile);
                        canReadWrite = !string.IsNullOrEmpty(contents);
                        System.IO.File.Delete(tmpFile);
                    }
                    catch (Exception)
                    {
                    }

                    //load file: GeoData-LastModified.txt
                    //if available then
                    //try to http head the data source url: http://download.geonames.org/export/dump/allCountries.zip
                    //pull off last-modified header
                    //if newer than the file last-modified, do a GET to download
                    //else
                    // do a get to download

                    bool shouldDownload = !System.IO.File.Exists(lastModifiedFile);
                    if (canReadWrite && System.IO.File.Exists(lastModifiedFile))
                    {
                        var lastModified = System.IO.File.ReadAllText(lastModifiedFile);
                        DateTime lastModifiedDate;
                        if (!DateTime.TryParse(lastModified, out lastModifiedDate))
                        {
                            shouldDownload = true;
                        }
                        else
                        {
                            var request = new WebClientRequest();
                            request.Headers = new WebHeaderCollection();
                            var client = new ParallelWebClient(request);
                            client.OpenReadTask(allCountriesUrl, "HEAD").ContinueWith(t =>
                            {
                                var headers = client.ResponseHeaders;
                                foreach (string h in headers.Keys)
                                {
                                    if (h.ToLowerInvariant() == "last-modified")
                                    {
                                        var header = headers[h];
                                        DateTime headerDate = DateTime.Now;
                                        DateTime.TryParse(header.ToString(), out headerDate);
                                        if (lastModifiedDate < headerDate)
                                        {
                                            shouldDownload = true;
                                        }
                                        break;
                                    }
                                }
                            }).Wait();
                        }
                    }

                    if (canReadWrite && shouldDownload)
                    {
                        var downloadTasks = new List <Task>();
                        var request = new WebClientRequest();
                        request.Headers = new WebHeaderCollection();
                        var client = new ParallelWebClient(request);

                        downloadTasks.Add(
                            client.DownloadData(allCountriesUrl).ContinueWith(t =>
                        {
                            if (System.IO.File.Exists(allCountriesFile))
                            {
                                System.IO.File.Delete(allCountriesFile);
                            }
                            System.IO.File.WriteAllBytes(allCountriesFile, t.Result);
                            var headers = client.ResponseHeaders;
                            foreach (string h in headers.Keys)
                            {
                                if (h.ToLowerInvariant() == "last-modified")
                                {
                                    var header = headers[h];
                                    DateTime headerDate = DateTime.Now;
                                    DateTime.TryParse(header.ToString(), out headerDate);
                                    System.IO.File.WriteAllText(lastModifiedFile, headerDate.ToString());
                                    break;
                                }
                            }
                        })
                            );

                        downloadTasks.Add(client.DownloadData(countryInfoUrl).ContinueWith(t =>
                        {
                            if (System.IO.File.Exists(countryInfoFile))
                            {
                                System.IO.File.Delete(countryInfoFile);
                            }
                            System.IO.File.WriteAllBytes(countryInfoFile, t.Result);
                        }));
                        downloadTasks.Add(client.DownloadData(featureCodes_enUrl).ContinueWith(t =>
                        {
                            if (System.IO.File.Exists(featureCodes_enFile))
                            {
                                System.IO.File.Delete(featureCodes_enFile);
                            }
                            System.IO.File.WriteAllBytes(featureCodes_enFile, t.Result);
                        }));
                        downloadTasks.Add(client.DownloadData(timeZonesUrl).ContinueWith(t =>
                        {
                            if (System.IO.File.Exists(timeZonesFile))
                            {
                                System.IO.File.Delete(timeZonesFile);
                            }
                            System.IO.File.WriteAllBytes(timeZonesFile, t.Result);
                        }));

                        Task.WaitAll(downloadTasks.ToArray());

                        //downloaded, now convert zip into serialized dat file

                        if (System.IO.File.Exists(allCountriesFile))
                        {
                            if (System.IO.File.Exists(countriesRawFile))
                            {
                                System.IO.File.Delete(countriesRawFile);
                            }
                            ICSharpCode.SharpZipLib.Zip.FastZip fz = new FastZip();
                            fz.ExtractZip(allCountriesFile, Root, FastZip.Overwrite.Always, null, null, null, true);
                            if (System.IO.File.Exists(countriesRawFile))
                            {
                                GeoData gd = GeoNameParser.ParseFile(countriesRawFile, timeZonesFile,
                                                                     featureCodes_enFile, countryInfoFile);

                                Serialize.SerializeBinaryToDisk(gd, DataFile);
                            }
                            if (System.IO.File.Exists(allCountriesFile))
                            {
                                System.IO.File.Delete(allCountriesFile);
                            }
                            if (System.IO.File.Exists(countriesRawFile))
                            {
                                System.IO.File.Delete(countriesRawFile);
                            }
                            if (System.IO.File.Exists(countryInfoFile))
                            {
                                System.IO.File.Delete(countryInfoFile);
                            }
                            if (System.IO.File.Exists(featureCodes_enFile))
                            {
                                System.IO.File.Delete(featureCodes_enFile);
                            }
                            if (System.IO.File.Exists(timeZonesFile))
                            {
                                System.IO.File.Delete(timeZonesFile);
                            }
                        }
                    }
                }
            }));
        }