public static void Run(List <string> locations, List <string> categories)
{
string yelp_api_key = ConfigurationManager.AppSettings.Get("yelp_api_key");
var yelpClient = new Yelp.Api.Client(yelp_api_key);
Database db = new Database();
DomainFinder df = new DomainFinder();
Dictionary <string, List <EmailDetails> > domainsEmails = new Dictionary <string, List <EmailDetails> >();
List <Details> details = new List <Details>();
foreach (string location in locations)
{
_logger.Info($"Exploring {location}");
try
{
foreach (string category in categories)
{
_logger.Info($" Category: {category}");
try
{
int maxResults = 50;
int offset = 0;
SearchResponse businesses = GetBusineesesFromYelp(category, maxResults, location, offset);
do
{
_logger.Info($" Found {businesses.Businesses?.Count ?? 0} businesses... offset {offset}");
if (businesses != null && businesses.Businesses != null)
{
foreach (var business in businesses.Businesses)
{
try
{
if (db.IsStoreExists(business.Name, business.Location.City, business.Location.State))
{
continue;
}
List <GoogleStoreModel> googleStores = new List <GoogleStoreModel>();
List <string> placeIdsByPhone = df.GetPlaceIdsByPhone(business.Phone);
if (placeIdsByPhone == null || placeIdsByPhone.Count == 0)
{
string str = $"{business.Name} {business.Location.City} {business.Location.State}";
List <string> placeIdsBySearchTerm = df.GetPlaceIdsBySearchTerm(str);
if (placeIdsBySearchTerm == null || placeIdsBySearchTerm.Count == 0)
{
InsertYelpDataToDB(business, StringConstants.STATUS_GOOD, db);
continue;
}
else
{
googleStores = GetGoogleStoresByIds(placeIdsBySearchTerm, df);
if (googleStores.Any(item => ArePhonesTheSame(business.Phone, item.Phone)))
{
googleStores = googleStores.Where(item => ArePhonesTheSame(business.Phone, item.Phone)).ToList();
}
}
}
else
{
googleStores = GetGoogleStoresByIds(placeIdsByPhone, df);
}
foreach (var googleStore in googleStores)
{
if (!IsWebsiteWorking(googleStore.Website))
{
InsertYelpAndGoogleDataToDB(null, null, null, null, null, business,
StringConstants.CATEGORY_STORE, StringConstants.STATUS_BAD, business.Phone, db);
continue;
}
else
{
string domain = df.GetDomainFromUrl(googleStore.Website);
_logger.Info($"GetSocialFromWebSite: {domain}");
GetSocialFromWebSite(domain, out string fb, out string instagram, out List <string> emailsList,
out string linkedin, out string twitter, out string phone);
googleStore.Facebook = fb;
googleStore.Instagram = instagram;
googleStore.Linkedin = linkedin;
googleStore.Twitter = twitter;
List <EmailDetails> emails = new List <EmailDetails>();
int numOfEMailsFromHunter = 0;
(emails, numOfEMailsFromHunter) = GetEmailsFromHunter(domain, googleStore.Name);
emails = emails.Where(email => IsEmailGood(email.Email)).ToList();
if (numOfEMailsFromHunter >= _numOfEmailsForChain)
{
InsertYelpAndGoogleDataToDB(domain, fb, instagram, linkedin, twitter, business,
StringConstants.CATEGORY_CHAIN, StringConstants.STATUS_GOOD, phone, db);
}
else
{
string sitePhone = string.IsNullOrEmpty(phone) ? googleStore.Phone : phone;
if (ArePhonesTheSame(business.Phone, sitePhone))
{
InsertYelpGoogleAndHunterDataToDB(business, googleStore, emails, StringConstants.CATEGORY_STORE, StringConstants.STATUS_GOOD, db);
}
else
{
InsertYelpDataToDB(business, StringConstants.STATUS_MEDIUM, db);
InsertGoogleHunterDataToDB(business, googleStore, emails, StringConstants.STATUS_MEDIUM, db);
}
}
}
}
}
catch (Exception ex)
{
_logger.Error(ex, "");
}
}
offset += 50;
businesses = GetBusineesesFromYelp(category, maxResults, location, offset);
}
}while ((businesses?.Businesses?.Count ?? 0) > 0);
}
catch (Exception x)
{ _logger.Error(x, ""); }
}
}
catch (Exception x)
{ _logger.Error(x, ""); }
}
_logger.Info("FINISH");
}
private static void Test()
{
string location = "New York, NY";
string category = "partysupplies";
var yelpResults = yelp.Program.GetBusineesesFromYelpSmallTypeStore(category, 50, location, 0);
DomainFinder df = new DomainFinder();
List <string> storesByNameNotByPhone = new List <string>();
List <string> storesByPhoneNotByName = new List <string>();
foreach (var business in yelpResults)
{
string searchTerm = $"{business.Name} {business.City} {business.State}";
List <string> placeIdsBySearchTerm = df.GetPlaceIdsBySearchTerm(searchTerm);
List <GoogleStoreModel> googleNameResults = new List <GoogleStoreModel>();
foreach (var place in placeIdsBySearchTerm)
{
var store = df.GetGoogleStoreByPlaceId(place);
googleNameResults.Add(store);
}
string phone = business.Phone;
var phoneStorePlaceIds = df.GetPlaceIdsByPhone(phone.Replace("+", ""));
List <GoogleStoreModel> googlePhoneRs = new List <GoogleStoreModel>();
foreach (var place in phoneStorePlaceIds)
{
var store = df.GetGoogleStoreByPlaceId(place);
googlePhoneRs.Add(store);
}
Console.WriteLine($"YELP - Store name: {business.Name} | City: {business.City} | State: {business.State}");
Console.WriteLine($"Google by search term: {googleNameResults.Count} items | Google by phone: {googlePhoneRs.Count} items");
if (googleNameResults == null || googleNameResults.Count == 0)
{
Console.WriteLine($"Google by search term - NOT FOUND!!!");
}
else
{
Console.WriteLine($"Google by search term results ({googleNameResults.Count} items):");
foreach (var place in googleNameResults)
{
Console.WriteLine($" Store name: {place.Name} | Phone: {place.Phone} | Website: {place.Website}");
}
}
if (googlePhoneRs == null || googlePhoneRs.Count == 0)
{
Console.WriteLine($"Google by phone - NOT FOUND!!!");
}
else
{
Console.WriteLine($"Google by phone results ({googlePhoneRs.Count} items):");
foreach (var place in googlePhoneRs)
{
Console.WriteLine($" Store name: {place.Name} | Phone: {place.Phone} | Website: {place.Website}");
}
}
if (googleNameResults != null && googleNameResults.Count > 0 &&
(googlePhoneRs == null || googlePhoneRs.Count == 0))
{
storesByNameNotByPhone.Add(business.Name);
}
if (googlePhoneRs != null && googlePhoneRs.Count > 0 &&
(googleNameResults == null || googleNameResults.Count == 0))
{
storesByPhoneNotByName.Add(business.Name);
}
}
Console.WriteLine($"************************************************************");
Console.WriteLine($"****************** *********************");
Console.WriteLine($"****************** Statistics *********************");
Console.WriteLine($"****************** *********************");
Console.WriteLine($"************************************************************");
Console.WriteLine();
Console.WriteLine($"Out of {yelpResults.Count} Businesses:");
Console.WriteLine($"Stores By Phone Not By Name: {storesByPhoneNotByName.Count} | Stores By Name Not By Phone: {storesByNameNotByPhone.Count}");
Console.Read();
Environment.Exit(1);
}
private static List <GoogleStoreModel> GetGoogleStoresByIds(List <string> placeIdsBySearchTerm, DomainFinder df)
{
List <GoogleStoreModel> ret = new List <GoogleStoreModel>();
foreach (var placeId in placeIdsBySearchTerm.Take(3))
{
var store = df.GetGoogleStoreByPlaceId(placeId);
ret.Add(store);
}
return(ret);
}