public List <string> CategoryPins(string CategoryName, int PageCount, ref PinInterestUser objPinUser) { List <string> lstPopularPins = new List <string>(); try { GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Start Getting Popular Pins For this User " + objPinUser.Username + " ]"); string BaseCategoryUrl = "http://pinterest.com/all/?category=" + CategoryName; for (int i = 1; i < PageCount; i++) { string CategoryUrl = "http://pinterest.com/all/?category=" + CategoryName + "&page=" + i; string PopularPinPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(CategoryUrl), BaseCategoryUrl, string.Empty, objPinUser.UserAgent); List <string> lst = globusRegex.GetHrefUrlTags(PopularPinPageSource); foreach (string item in lst) { if (item.Contains("/pin/") && !item.Contains("edit")) { try { int FirstPinPoint = item.IndexOf("/pin/"); int SecondPinPoint = item.IndexOf("class="); string PinUrl = item.Substring(FirstPinPoint, SecondPinPoint - FirstPinPoint).Replace("\"", string.Empty).Replace("pin", string.Empty).Replace("/", string.Empty).Trim(); lstPopularPins.Add(PinUrl); } catch (Exception ex) { GlobusLogHelper.log.Error(" Error : " + ex.StackTrace); } } } } lstPopularPins = lstPopularPins.Distinct().ToList(); lstPopularPins.Reverse(); GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Total Pin Urls Collected " + lstPopularPins.Count + " ]"); } catch (Exception ex) { GlobusLogHelper.log.Error(" Error : " + ex.StackTrace); } return(lstPopularPins); }
public List <string> getUserKeywordSearch_New(string keyword, int Count, ref PinInterestUser objPinUser) { List <string> lstUserPins = new List <string>(); try { GlobusLogHelper.log.Info(" => [ Start Getting Pins For the Keyword " + keyword + " ]"); string UserUrl = string.Empty; string UserPins = string.Empty; string Board = string.Empty; string Pins = string.Empty; string Like = string.Empty; string Follower = string.Empty; string Following = string.Empty; for (int i = 0; i <= Count; i++) { try { if (i == 0) { UserUrl = "http://pinterest.com/search/people/?q=" + keyword; } else { UserUrl = "http://pinterest.com/resource/SearchResource/get/?source_url=%2Fsearch%2Fpins%2F%3Fq%3Droses%26rs%3Dac%26len%3D4&data=%7B%22options%22%3A%7B%22query%22%3A%22roses%22%2C%22bookmarks%22%3A%5B%22b281MHw4ODgyNzJiYWQxN2E0NmM2MTNlNjNkYjY1Y2E4ZDY2OTI4ODZkMzBiNGEyNjU4ZGMyODFmMzhhOTUzZjE2NTRm%22%5D%2C%22show_scope_selector%22%3Atrue%2C%22scope%22%3A%22pins%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22411bd12%22%7D%2C%22module%22%3A%7B%22name%22%3A%22GridItems%22%2C%22options%22%3A%7B%22scrollable%22%3Atrue%2C%22show_grid_footer%22%3Atrue%2C%22centered%22%3Atrue%2C%22reflow_all%22%3Atrue%2C%22virtualize%22%3Atrue%2C%22item_options%22%3A%7B%22show_pinner%22%3Atrue%2C%22show_pinned_from%22%3Afalse%2C%22show_board%22%3Atrue%7D%2C%22layout%22%3A%22variable_height%22%7D%7D%2C%22append%22%3Atrue%2C%22error_strategy%22%3A1%7D&module_path=App()%3EHeader()%3Eui.SearchForm()%3Eui.TypeaheadField(enable_recent_queries%3Dtrue%2C+name%3Dq%2C+view_type%3Dsearch%2C+prefetch_on_focus%3Dtrue%2C+value%3D%22%22%2C+populate_on_result_highlight%3Dtrue%2C+search_delay%3D0%2C+search_on_focus%3Dtrue%2C+placeholder%3DSearch%2C+tags%3Dautocomplete)&_=" + DateTime.Now.Ticks; } string UserPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(UserUrl), "http://pinterest.com/", string.Empty, objPinUser.UserAgent); if (!UserPageSource.Contains("The page you're looking for could not be found")) { List <string> lst = globusRegex.GetHrefUrlTags(UserPageSource); string[] array = System.Text.RegularExpressions.Regex.Split(UserPageSource, "href="); foreach (string item in array) { if (item.Contains("class=\\\"userWrapper\\\"")) { try { int FirstPinPoint = item.IndexOf("\\\"/"); int SecondPinPoint = item.IndexOf("/\\\""); string PinUrl = item.Substring(FirstPinPoint, SecondPinPoint - FirstPinPoint).Replace("\\", "").Replace("\"", string.Empty).Replace("href=", string.Empty).Replace("/", string.Empty).Trim(); string FollowersUrl = "https://pinterest.com/" + PinUrl; string GetResponse = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(FollowersUrl)); #region ChkboxFollowCertainAmtofValue if (ChkboxFollowCertainAmtofValue == true) { if (!GetResponse.Contains("<div>Something went wrong!</div>")) { try { try { Board = Utils.Utils.getBetween(GetResponse, "pinterestapp:boards\" content=\"", "\" data-app>"); } catch (Exception ex) { Board = "NA"; GlobusLogHelper.log.Error(" Error ==>" + ex.Message); } try { Pins = Utils.Utils.getBetween(GetResponse, "pinterestapp:pins\" content=\"", "\" data-app>"); } catch (Exception ex) { Pins = "NA"; GlobusLogHelper.log.Error(" Error ==>" + ex.Message); } try { Like = Utils.Utils.getBetween(GetResponse, "like_count\":", ",\""); } catch (Exception ex) { Like = "NA"; GlobusLogHelper.log.Error(" Error ==>" + ex.Message); } try { Follower = Utils.Utils.getBetween(GetResponse, "pinterestapp:followers\" content=\"", "\" data-app>"); } catch (Exception ex) { Follower = "NA"; GlobusLogHelper.log.Error(" Error ==>" + ex.Message); } try { Following = Utils.Utils.getBetween(GetResponse, "pinterestapp:following\" content=\"", "\" data-app>"); } catch (Exception ex) { Following = "NA"; GlobusLogHelper.log.Error(" Error ==>" + ex.Message); } } catch (Exception ex) { GlobusLogHelper.log.Error(" Error ==>" + ex.Message); } if (chkBoard == true) { if ((minValueBoard <= int.Parse(Board)) && (int.Parse(Board) <= maxValueBoard)) { } else { continue; } } if (chkPin == true) { if ((minValuePin <= int.Parse(Pins)) && (int.Parse(Pins) <= maxValuePin)) { } else { continue; } } if (chkLike == true) { if ((minValueLike <= int.Parse(Like)) && (int.Parse(Like) <= maxValueLike)) { } else { continue; } } if (chkFollower == true) { if ((minValueFollower <= int.Parse(Follower)) && (int.Parse(Follower) <= maxValueFollower)) { } else { continue; } } if (chkFollowing == true) { if ((minValueFollowing <= int.Parse(Following)) && (int.Parse(Following) <= maxValueFollowing)) { } else { continue; } } lstUserPins.Add(PinUrl); } } #endregion ChkboxFollowCertainAmtofValue else { lstUserPins.Add(PinUrl); } if (lstUserPins.Count == NoOfUserFollowByKeyword) { break; } } catch (Exception ex) { GlobusLogHelper.log.Error(" Error ==>" + ex.Message); } } } } } catch (Exception ex) { GlobusLogHelper.log.Info(" => [ Sorry No More Pages ]"); break; } lstUserPins = lstUserPins.Distinct().ToList(); lstUserPins.Reverse(); } GlobusLogHelper.log.Info(" => [ Total Pin Urls Collected " + lstUserPins.Count + " ]"); } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } return(lstUserPins); }
public List <string> GetUserFollower_new() { try { ClGlobul.lstTotalUserScraped.Clear(); List <string> lstFollower = new List <string>(); GlobusLogHelper.log.Info(" => [ Starting Extraction Of Followers For " + UserName + " ]"); objPinUser.globusHttpHelper = new GlobusHttpHelper(); string TotalFollowersUrl = "https://pinterest.com/" + UserName; string responseFollowersUrl = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(TotalFollowersUrl), referer, string.Empty, ""); int TotalFollower = int.Parse(Utils.Utils.getBetween(responseFollowersUrl, "follower_count\": ", ",")); int NoOfPage = TotalFollower / 12 + 1; for (int i = 1; i <= NoOfPage; i++) // for (int i = 1; i <= NoOfPage; i++) { try { if (i == 1) { FollowUrl = "http://pinterest.com/" + UserName + "/" + UserScraperType + "/"; FollowerPageSource = FollowerPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(FollowUrl), referer, string.Empty, ""); referer = FollowUrl; } else { try { FollowUrl = "http://pinterest.com/resource/UserFollowersResource/get/?source_url=%2F" + UserName + "%2Ffollowers%2F&data=%7B%22options%22%3A%7B%22username%22%3A%22" + UserName + "%22%2C%22bookmarks%22%3A%5B%22" + bookmark + "%22%5D%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + AppVersion + "%22%7D%2C%22module%22%3A%7B%22name%22%3A%22GridItems%22%2C%22options%22%3A%7B%22scrollable%22%3Atrue%2C%22show_grid_footer%22%3Afalse%2C%22centered%22%3Atrue%2C%22reflow_all%22%3Atrue%2C%22virtualize%22%3Atrue%2C%22layout%22%3A%22fixed_height%22%7D%7D%2C%22append%22%3Atrue%2C%22error_strategy%22%3A" + i + "%7D&_=" + DateTime.Now.Ticks; FollowerPageSource = objPinUser.globusHttpHelper.getHtmlfromUrlProxy(new Uri(FollowUrl), referer, "", 80, string.Empty, "", ""); } catch (Exception ex) { GlobusLogHelper.log.Error(" Error : " + ex.StackTrace); } } ///Get App Version if (FollowerPageSource.Contains("app_version") && string.IsNullOrEmpty(AppVersion)) { string[] ArrAppVersion = System.Text.RegularExpressions.Regex.Split(FollowerPageSource, "app_version"); if (ArrAppVersion.Count() > 0) { string DataString = ArrAppVersion[ArrAppVersion.Count() - 1]; int startindex = DataString.IndexOf("\": \""); int endindex = DataString.IndexOf("\", \""); AppVersion = DataString.Substring(startindex, endindex - startindex).Replace("\": \"", ""); } } try { if (!FollowerPageSource.Contains("No one has followed")) { List <string> lst = objGlobusRegex.GetHrefUrlTags(FollowerPageSource); if (lst.Count == 0) { lst = System.Text.RegularExpressions.Regex.Split(FollowerPageSource, "href").ToList(); } foreach (string item in lst) { if (item.Contains("class=\"userWrapper") || item.Contains("class=\\\"userWrapper")) { try { string User = string.Empty; if (item.Contains("\\")) { int FirstPinPoint = item.IndexOf("=\\\""); int SecondPinPoint = item.IndexOf("class="); User = item.Substring(FirstPinPoint, SecondPinPoint - FirstPinPoint).Replace("\"", string.Empty).Replace("\\", string.Empty).Replace("=", string.Empty).Replace("/", string.Empty).Trim(); } else { int FirstPinPoint = item.IndexOf("href="); int SecondPinPoint = item.IndexOf("class="); User = item.Substring(FirstPinPoint, SecondPinPoint - FirstPinPoint).Replace("\"", string.Empty).Replace("href=", string.Empty).Replace("/", string.Empty).Trim(); } if (followers.Count == MaxCountScraper) { return(ClGlobul.lstTotalUserScraped); } // GlobusLogHelper.log.Info(" => [ " + User + " ]"); followers.Add(User); if (followers.Count == MaxCountScraper) { break; } } catch (Exception ex) { GlobusLogHelper.log.Error(" Error : " + ex.StackTrace); } } } ///get bookmarks value from page /// if (FollowerPageSource.Contains("bookmarks")) { string[] bookmarksDataArr = System.Text.RegularExpressions.Regex.Split(FollowerPageSource, "bookmarks"); string Datavalue = string.Empty; if (bookmarksDataArr.Count() > 2) { Datavalue = bookmarksDataArr[bookmarksDataArr.Count() - 2]; } else { Datavalue = bookmarksDataArr[bookmarksDataArr.Count() - 1]; } bookmark = Datavalue.Substring(Datavalue.IndexOf(": [\"") + 4, Datavalue.IndexOf("]") - Datavalue.IndexOf(": [\"") - 5); } followers = followers.Distinct().ToList(); if (followers.Count >= MaxCountScraper) { foreach (string lstdata in followers) { lstFollower.Add(lstdata); #region AccountReport string module = "Scraper"; string status = "Followers"; qm.insertAccReportScrapeUser(module, lstdata, status, DateTime.Now); objScraperDelegate(); #endregion if (lstFollower.Count >= MaxCountScraper) { return(ClGlobul.lstTotalUserScraped); } } ClGlobul.lstTotalUserScraped = lstFollower.Distinct().ToList(); Thread.Sleep(1000); } } else { GlobusLogHelper.log.Info(" => [ No Followers ]"); break; } } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } } } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } GlobusLogHelper.log.Info(" => [ Total Followers : " + ClGlobul.lstTotalUserScraped.Count + " ]"); return(ClGlobul.lstTotalUserScraped); }
public List <string> GetUserFollowers_new(string UserName, int UserFollowerCount, ref PinInterestUser objPinUser) { try { int count = 0; List <string> Followers = new List <string>(); List <string> TotalScraperFollowers = new List <string>(); GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Scraping Followers ] "); objPinUser.globusHttpHelper = new GlobusHttpHelper(); for (int i = 1; i <= 100; i++) { try { string FollowerPageSource = string.Empty; if (i == 1) { FollowUrl = "http://www.pinterest.com/" + UserName + "/followers/"; FollowerPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(FollowUrl), referer, string.Empty, ""); referer = FollowUrl; } else { FollowUrl = "https://www.pinterest.com/resource/UserFollowersResource/get/?source_url=%2F" + UserName + "%2Ffollowers%2F&data=%7B%22options%22%3A%7B%22username%22%3A%22" + UserName + "%22%2C%22bookmarks%22%3A%5B%22" + bookmark + "%3D%3D%22%5D%7D%2C%22context%22%3A%7B%7D%7D&module_path=App%3EUserProfilePage%3EUserInfoBar(tab%3Dfollowers%2C+spinner%3D%5Bobject+Object%5D%2C+resource%3DUserResource(username%3DSimplyBags%2C+invite_code%3Dnull))&_="; try { FollowerPageSource = objPinUser.globusHttpHelper.getHtmlfromUrlProxy(new Uri(FollowUrl), referer, "", 80, string.Empty, "", ""); } catch { FollowerPageSource = objPinUser.globusHttpHelper.getHtmlfromUrlProxy(new Uri(FollowUrl), "", Convert.ToInt32(""), "", ""); } if (FollowerPageSource.Contains("Whoops! We couldn't find that page.")) { break; } } ///Get App Version if (FollowerPageSource.Contains("app_version") && string.IsNullOrEmpty(AppVersion)) { string[] ArrAppVersion = System.Text.RegularExpressions.Regex.Split(FollowerPageSource, "app_version"); if (ArrAppVersion.Count() > 0) { string DataString = ArrAppVersion[ArrAppVersion.Count() - 1]; int startindex = DataString.IndexOf("\": \""); int endindex = DataString.IndexOf("\", \""); AppVersion = DataString.Substring(startindex, endindex - startindex).Replace("\": \"", ""); } } ///get bookmarks value from page /// if (FollowerPageSource.Contains("bookmarks")) { string[] bookmarksDataArr = System.Text.RegularExpressions.Regex.Split(FollowerPageSource, "bookmarks"); string Datavalue = string.Empty; if (bookmarksDataArr.Count() > 2) { Datavalue = bookmarksDataArr[bookmarksDataArr.Count() - 2]; } else { Datavalue = bookmarksDataArr[bookmarksDataArr.Count() - 1]; } bookmark = Datavalue.Substring(Datavalue.IndexOf(": [\"") + 4, Datavalue.IndexOf("]") - Datavalue.IndexOf(": [\"") - 5); } try { if (!FollowerPageSource.Contains("No one has followed")) { List <string> lst = globusRegex.GetHrefUrlTags(FollowerPageSource); string[] arraydata = System.Text.RegularExpressions.Regex.Split(FollowerPageSource, "href="); if (lst.Count == 0) { lst = System.Text.RegularExpressions.Regex.Split(FollowerPageSource, "href").ToList(); } foreach (string item in arraydata) { if (item.Contains("class=\"userWrapper") || item.Contains("class=\\\"userWrapper")) { try { if (item.Contains("\\")) { int FirstPinPoint = item.IndexOf("\\\"/"); int SecondPinPoint = item.IndexOf("/\\\""); User = item.Substring(FirstPinPoint, SecondPinPoint - FirstPinPoint).Replace("\"", string.Empty).Replace("\\", string.Empty).Replace("=", string.Empty).Replace("/", string.Empty).Trim(); } else { int FirstPinPoint = item.IndexOf("\\\"/"); int SecondPinPoint = item.IndexOf("/\\\""); User = item.Substring(FirstPinPoint, SecondPinPoint - FirstPinPoint).Replace("\"", string.Empty).Replace("href=", string.Empty).Replace("/", string.Empty).Trim(); } // GlobusLogHelper.log.Info(" => [ " + User + " ]"); Followers.Add(User); } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } } else if (item.Contains("[{\"username\":") || item.Contains("[{\"username\":")) { try { string[] arrayUsername = System.Text.RegularExpressions.Regex.Split(item, "\"username\": "); arrayUsername = arrayUsername.Skip(1).ToArray(); foreach (string itemUsername in arrayUsername) { if (itemUsername.Contains("\"domain_verified\"")) { User = Utils.Utils.getBetween(itemUsername, "\"", "\", \"domain_verified\""); Followers.Add(User); } } } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } } } Followers = Followers.Distinct().ToList(); foreach (string lstdata in Followers) { TotalScraperFollowers.Add(lstdata); count++; } TotalScraperFollowers = TotalScraperFollowers.Distinct().ToList(); Thread.Sleep(1000); } else { GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ No Followers ]"); break; } } catch (Exception ex) { GlobusLogHelper.log.Error("'Error :" + ex.StackTrace); } } catch (Exception ex) { GlobusLogHelper.log.Error("'Error :" + ex.StackTrace); } } objPinUser.lstUserFollowers = TotalScraperFollowers; } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Total Followers : " + lstTotalUserScraped.Count + " ]"); return(objPinUser.lstUserFollowers); }
public List <string> getUserKeywordSearch_New(string keyword, int Count, ref PinInterestUser objPinUser) { List <string> lstUserPins = new List <string>(); try { GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Start Getting Pins For the Keyword " + keyword + " ]"); string UserUrl = string.Empty; string UserPins = string.Empty; for (int i = 0; i <= Count; i++) { try { if (i == 0) { UserUrl = "http://pinterest.com/search/people/?q=" + keyword; } else { UserUrl = "http://pinterest.com/resource/SearchResource/get/?source_url=%2Fsearch%2Fpins%2F%3Fq%3Droses%26rs%3Dac%26len%3D4&data=%7B%22options%22%3A%7B%22query%22%3A%22roses%22%2C%22bookmarks%22%3A%5B%22b281MHw4ODgyNzJiYWQxN2E0NmM2MTNlNjNkYjY1Y2E4ZDY2OTI4ODZkMzBiNGEyNjU4ZGMyODFmMzhhOTUzZjE2NTRm%22%5D%2C%22show_scope_selector%22%3Atrue%2C%22scope%22%3A%22pins%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22411bd12%22%7D%2C%22module%22%3A%7B%22name%22%3A%22GridItems%22%2C%22options%22%3A%7B%22scrollable%22%3Atrue%2C%22show_grid_footer%22%3Atrue%2C%22centered%22%3Atrue%2C%22reflow_all%22%3Atrue%2C%22virtualize%22%3Atrue%2C%22item_options%22%3A%7B%22show_pinner%22%3Atrue%2C%22show_pinned_from%22%3Afalse%2C%22show_board%22%3Atrue%7D%2C%22layout%22%3A%22variable_height%22%7D%7D%2C%22append%22%3Atrue%2C%22error_strategy%22%3A1%7D&module_path=App()%3EHeader()%3Eui.SearchForm()%3Eui.TypeaheadField(enable_recent_queries%3Dtrue%2C+name%3Dq%2C+view_type%3Dsearch%2C+prefetch_on_focus%3Dtrue%2C+value%3D%22%22%2C+populate_on_result_highlight%3Dtrue%2C+search_delay%3D0%2C+search_on_focus%3Dtrue%2C+placeholder%3DSearch%2C+tags%3Dautocomplete)&_=" + DateTime.Now.Ticks; } string UserPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(UserUrl), "http://pinterest.com/", string.Empty, objPinUser.UserAgent); if (!UserPageSource.Contains("The page you're looking for could not be found")) { List <string> lst = globusRegex.GetHrefUrlTags(UserPageSource); string[] array = System.Text.RegularExpressions.Regex.Split(UserPageSource, "href="); foreach (string item in array) { if (item.Contains("class=\\\"userWrapper\\\"")) { try { int FirstPinPoint = item.IndexOf("\\\"/"); int SecondPinPoint = item.IndexOf("/\\\""); string PinUrl = item.Substring(FirstPinPoint, SecondPinPoint - FirstPinPoint).Replace("\\", "").Replace("\"", string.Empty).Replace("href=", string.Empty).Replace("/", string.Empty).Trim(); lstUserPins.Add(PinUrl); } catch (Exception ex) { } } } } } catch (Exception ex) { GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Sorry No More Pages ]"); break; } lstUserPins = lstUserPins.Distinct().ToList(); lstUserPins.Reverse(); } GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Total Pin Urls Collected " + lstUserPins.Count + " ]"); } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } return(lstUserPins); }