public ClosestCityFinder( string citiesPath, string alternateNamesPath, string admin1Path, string admin2Path, string countriesPath, string clliPath, string unlocodePath) { var alternateNamesDict = GeonamesAlternateNamesParser.ParseToDict(alternateNamesPath); var admin1Dict = GeonamesAdminParser.ParseToDict(admin1Path); var admin2Dict = GeonamesAdminParser.ParseToDict(admin2Path); var countryEntities = GeonamesCountriesParser.ParseToList(countriesPath); var countryCodesDict = GeonamesCountriesParser.ListToISOCodeDict(countryEntities); var geonamesIdsToCLLICodes = CLLICodesParser.ParseToDict(clliPath); var geonamesIdsToUNLOCODECodes = UNLOCODECodesParser.ParseToDict(unlocodePath); this.GeohashesToCities = new Dictionary <string, List <GeonamesCityEntity> >(); foreach (var entity in GeonamesCitiesParser.Parse(citiesPath, alternateNamesDict, admin1Dict, admin2Dict, countryCodesDict, geonamesIdsToCLLICodes, geonamesIdsToUNLOCODECodes)) { var geohash = GeoHash.Encode(entity.Latitude, entity.Longitude, numberOfChars: 3); // 3 = ±78km List <GeonamesCityEntity> citiesInGeohash; if (!this.GeohashesToCities.TryGetValue(geohash, out citiesInGeohash)) { citiesInGeohash = new List <GeonamesCityEntity>(); this.GeohashesToCities[geohash] = citiesInGeohash; } citiesInGeohash.Add(entity); } }
public CityFeaturesAggregator( string citiesPath, string alternateNamesPath, string admin1Path, string admin2Path, string countriesPath, string clliPath, string unlocodePath, List <CityFeaturesGenerator> cityFeatureGenerators = null, List <AddOnCityFeaturesGenerator> cityAddOnFeatureGenerators = null, List <CityFeaturesGenerator> externalCityFeatureGenerators = null, List <AddOnCityFeaturesGenerator> externalCityAddOnFeatureGenerators = null, FeaturesConfig featuresConfig = null) : this(cityFeatureGenerators, cityAddOnFeatureGenerators, externalCityFeatureGenerators, externalCityAddOnFeatureGenerators, featuresConfig) { var alternateNamesDict = GeonamesAlternateNamesParser.ParseToDict(alternateNamesPath); var admin1Dict = GeonamesAdminParser.ParseToDict(admin1Path); var admin2Dict = GeonamesAdminParser.ParseToDict(admin2Path); var countryEntities = GeonamesCountriesParser.ParseToList(countriesPath); var countryCodesDict = GeonamesCountriesParser.ListToISOCodeDict(countryEntities); var geonamesIdsToCLLICodes = CLLICodesParser.ParseToDict(clliPath); Dictionary <int, HashSet <string> > geonamesIdsToUNLOCODECodes = null; // Needed if enabling ExactUNLOCODEFeaturesGenerator below /* * if (unlocodePath != null) * { * geonamesIdsToUNLOCODECodes = UNLOCODECodesParser.ParseToDict(unlocodePath); * } */ var total = 0; var withPop = 0; foreach (var entity in GeonamesCitiesParser.Parse(citiesPath, alternateNamesDict, admin1Dict, admin2Dict, countryCodesDict, geonamesIdsToCLLICodes, geonamesIdsToUNLOCODECodes)) { total++; if (this.featuresConfig.MinimumPopulation == 0 || entity.Population >= this.featuresConfig.MinimumPopulation) { withPop++; foreach (var cityFeatureGenerator in this.cityFeatureGenerators) { cityFeatureGenerator.IngestCityEntity(entity); } } if (total % 1000 == 0) { Console.WriteLine($"Loading cities - total: {total}, withPop: {withPop}"); } } /* * foreach (var cityFeatureGenerator in this.cityFeatureGenerators) * { * try * { * Console.WriteLine($"Estimated bytes for {cityFeatureGenerator.GetType().FullName}: {FeatureUtils.EstimateObjectSizeInBytes(cityFeatureGenerator)}"); * } * catch (Exception) * { * Console.WriteLine($"Could not estimate bytes for {cityFeatureGenerator.GetType().FullName}"); * } * } */ }