private static void Test() { string location = "New York, NY"; string category = "partysupplies"; var yelpResults = yelp.Program.GetBusineesesFromYelpSmallTypeStore(category, 50, location, 0); DomainFinder df = new DomainFinder(); List <string> storesByNameNotByPhone = new List <string>(); List <string> storesByPhoneNotByName = new List <string>(); foreach (var business in yelpResults) { string searchTerm = $"{business.Name} {business.City} {business.State}"; List <string> placeIdsBySearchTerm = df.GetPlaceIdsBySearchTerm(searchTerm); List <GoogleStoreModel> googleNameResults = new List <GoogleStoreModel>(); foreach (var place in placeIdsBySearchTerm) { var store = df.GetGoogleStoreByPlaceId(place); googleNameResults.Add(store); } string phone = business.Phone; var phoneStorePlaceIds = df.GetPlaceIdsByPhone(phone.Replace("+", "")); List <GoogleStoreModel> googlePhoneRs = new List <GoogleStoreModel>(); foreach (var place in phoneStorePlaceIds) { var store = df.GetGoogleStoreByPlaceId(place); googlePhoneRs.Add(store); } Console.WriteLine($"YELP - Store name: {business.Name} | City: {business.City} | State: {business.State}"); Console.WriteLine($"Google by search term: {googleNameResults.Count} items | Google by phone: {googlePhoneRs.Count} items"); if (googleNameResults == null || googleNameResults.Count == 0) { Console.WriteLine($"Google by search term - NOT FOUND!!!"); } else { Console.WriteLine($"Google by search term results ({googleNameResults.Count} items):"); foreach (var place in googleNameResults) { Console.WriteLine($" Store name: {place.Name} | Phone: {place.Phone} | Website: {place.Website}"); } } if (googlePhoneRs == null || googlePhoneRs.Count == 0) { Console.WriteLine($"Google by phone - NOT FOUND!!!"); } else { Console.WriteLine($"Google by phone results ({googlePhoneRs.Count} items):"); foreach (var place in googlePhoneRs) { Console.WriteLine($" Store name: {place.Name} | Phone: {place.Phone} | Website: {place.Website}"); } } if (googleNameResults != null && googleNameResults.Count > 0 && (googlePhoneRs == null || googlePhoneRs.Count == 0)) { storesByNameNotByPhone.Add(business.Name); } if (googlePhoneRs != null && googlePhoneRs.Count > 0 && (googleNameResults == null || googleNameResults.Count == 0)) { storesByPhoneNotByName.Add(business.Name); } } Console.WriteLine($"************************************************************"); Console.WriteLine($"****************** *********************"); Console.WriteLine($"****************** Statistics *********************"); Console.WriteLine($"****************** *********************"); Console.WriteLine($"************************************************************"); Console.WriteLine(); Console.WriteLine($"Out of {yelpResults.Count} Businesses:"); Console.WriteLine($"Stores By Phone Not By Name: {storesByPhoneNotByName.Count} | Stores By Name Not By Phone: {storesByNameNotByPhone.Count}"); Console.Read(); Environment.Exit(1); }
public static void Run(List <string> locations, List <string> categories) { string yelp_api_key = ConfigurationManager.AppSettings.Get("yelp_api_key"); var yelpClient = new Yelp.Api.Client(yelp_api_key); Database db = new Database(); DomainFinder df = new DomainFinder(); Dictionary <string, List <EmailDetails> > domainsEmails = new Dictionary <string, List <EmailDetails> >(); List <Details> details = new List <Details>(); foreach (string location in locations) { _logger.Info($"Exploring {location}"); try { foreach (string category in categories) { _logger.Info($" Category: {category}"); try { int maxResults = 50; int offset = 0; SearchResponse businesses = GetBusineesesFromYelp(category, maxResults, location, offset); do { _logger.Info($" Found {businesses.Businesses?.Count ?? 0} businesses... offset {offset}"); if (businesses != null && businesses.Businesses != null) { foreach (var business in businesses.Businesses) { try { if (db.IsStoreExists(business.Name, business.Location.City, business.Location.State)) { continue; } List <GoogleStoreModel> googleStores = new List <GoogleStoreModel>(); List <string> placeIdsByPhone = df.GetPlaceIdsByPhone(business.Phone); if (placeIdsByPhone == null || placeIdsByPhone.Count == 0) { string str = $"{business.Name} {business.Location.City} {business.Location.State}"; List <string> placeIdsBySearchTerm = df.GetPlaceIdsBySearchTerm(str); if (placeIdsBySearchTerm == null || placeIdsBySearchTerm.Count == 0) { InsertYelpDataToDB(business, StringConstants.STATUS_GOOD, db); continue; } else { googleStores = GetGoogleStoresByIds(placeIdsBySearchTerm, df); if (googleStores.Any(item => ArePhonesTheSame(business.Phone, item.Phone))) { googleStores = googleStores.Where(item => ArePhonesTheSame(business.Phone, item.Phone)).ToList(); } } } else { googleStores = GetGoogleStoresByIds(placeIdsByPhone, df); } foreach (var googleStore in googleStores) { if (!IsWebsiteWorking(googleStore.Website)) { InsertYelpAndGoogleDataToDB(null, null, null, null, null, business, StringConstants.CATEGORY_STORE, StringConstants.STATUS_BAD, business.Phone, db); continue; } else { string domain = df.GetDomainFromUrl(googleStore.Website); _logger.Info($"GetSocialFromWebSite: {domain}"); GetSocialFromWebSite(domain, out string fb, out string instagram, out List <string> emailsList, out string linkedin, out string twitter, out string phone); googleStore.Facebook = fb; googleStore.Instagram = instagram; googleStore.Linkedin = linkedin; googleStore.Twitter = twitter; List <EmailDetails> emails = new List <EmailDetails>(); int numOfEMailsFromHunter = 0; (emails, numOfEMailsFromHunter) = GetEmailsFromHunter(domain, googleStore.Name); emails = emails.Where(email => IsEmailGood(email.Email)).ToList(); if (numOfEMailsFromHunter >= _numOfEmailsForChain) { InsertYelpAndGoogleDataToDB(domain, fb, instagram, linkedin, twitter, business, StringConstants.CATEGORY_CHAIN, StringConstants.STATUS_GOOD, phone, db); } else { string sitePhone = string.IsNullOrEmpty(phone) ? googleStore.Phone : phone; if (ArePhonesTheSame(business.Phone, sitePhone)) { InsertYelpGoogleAndHunterDataToDB(business, googleStore, emails, StringConstants.CATEGORY_STORE, StringConstants.STATUS_GOOD, db); } else { InsertYelpDataToDB(business, StringConstants.STATUS_MEDIUM, db); InsertGoogleHunterDataToDB(business, googleStore, emails, StringConstants.STATUS_MEDIUM, db); } } } } } catch (Exception ex) { _logger.Error(ex, ""); } } offset += 50; businesses = GetBusineesesFromYelp(category, maxResults, location, offset); } }while ((businesses?.Businesses?.Count ?? 0) > 0); } catch (Exception x) { _logger.Error(x, ""); } } } catch (Exception x) { _logger.Error(x, ""); } } _logger.Info("FINISH"); }