public void Start_Action_UserFollower(ref InstagramUser obj_Gram) { try { lstThreadsUserFollower.Add(Thread.CurrentThread); lstThreadsUserFollower.Distinct(); Thread.CurrentThread.IsBackground = true; } catch (Exception ex) { GlobusLogHelper.log.Error("Error : " + ex.StackTrace); } try { string[] lst_UserFollower_listedUser = ClGlobul.lstUserFollower_Username.ToArray(); GlobusHttpHelper obj_globul = new GlobusHttpHelper(); bool value = false; string proxyAddress = string.Empty; string proxyport = "80"; string proxyusername = string.Empty; string proxypasswrd = string.Empty; List<string> lst_workingfinalproxy = new List<string>(); string[] lst_Userfollowerlist = {}; try { DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy"); DataTable dt = ds.Tables[0]; foreach (DataRow item in dt.Rows) { lst_workingfinalproxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]); } } catch (Exception ex) { } try { if (lst_workingfinalproxy.Count > 0) { string res_secondURL = obj_Gram.globusHttpHelper.getHtmlfromUrl_withProxt(new Uri("https://www.instagram.com"), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); string proxy = lst_workingfinalproxy[RandomNumberGenerator.GenerateRandom(0, lst_workingfinalproxy.Count)]; string[] proxydata = Regex.Split(proxy, ":"); if (proxydata.Count() == 2) { proxyAddress = proxydata[0]; proxyport = proxydata[1]; } if (proxydata.Count() == 4) { proxyAddress = proxydata[0]; proxyport = proxydata[1]; proxyusername = proxydata[2]; proxypasswrd = proxydata[3]; } } } catch (Exception ex) { } foreach(string username in lst_UserFollower_listedUser) { string gramfeed_user_Url = "http://www.gramfeed.com/" + username; // Username_ScrapFollower = itemusername; List<string> follower_list = new List<string>(); int count = 0; ClGlobul.switchAccount = false; string GramFeed_Userhit = obj_globul.getHtmlfromUrl_withProxt(new Uri(gramfeed_user_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&"); string Id_Url = "https://api.instagram.com/v1/users/search?q=" + username + "&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024"; string Gram_Hit_second = obj_globul.getHtmlfromUrl_withProxt(new Uri(Id_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); if (string.IsNullOrEmpty(Gram_Hit_second)) { Thread.Sleep(1 * 3000); Gram_Hit_second = obj_ChangeProxy.chnageproxyMethod(Id_Url); } string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "\"id\":", "\",").Replace("\"", "").Replace(" ", ""); string Follower_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/followed-by?client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024"; string GramFeed_UserFollower = obj_globul.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); if (string.IsNullOrEmpty(GramFeed_UserFollower)) { if (lst_workingfinalproxy.Count > 0) { GramFeed_UserFollower = obj_ChangeProxy.chnageproxyMethod(Follower_Url); } else { Thread.Sleep(1 * 3000); GramFeed_UserFollower = obj_globul.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); } } string[] follower_regexlist = Regex.Split(GramFeed_UserFollower, "username"); foreach (string item in follower_regexlist) { if (item.Contains("profile_picture")) { string user_name = Utils.getBetween(item, "\":", "\",").Replace("\"", "").Replace(" ", ""); if (follower_list.Count < Number_Follower && !follower_list.Contains(user_name)) { follower_list.Add(user_name); follower_list = follower_list.Distinct().ToList(); ClGlobul.lst_follower_UserFollower.Add(user_name); count++; try { GlobusLogHelper.log.Info(obj_Gram.username +" --> Scraped ===> " + user_name); } catch (Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } } else { break; } } } if (UserFollower_Delaymin != 0) { mindelay = UserFollower_Delaymin; } if (UserFollower_Delaymax != 0) { maxdelay = UserFollower_Delaymax; } int delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay); GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds "); Thread.Sleep(delay * 1000); if (GramFeed_UserFollower.Contains("next_url")) { value = true; while (value) { if (GramFeed_UserFollower.Contains("next_url") && follower_list.Count < Number_Follower) { string hit_data = Utils.getBetween(GramFeed_UserFollower, "next_url\":", "\",").Replace("\u0026", "&").Replace("\\", "").Replace("u0026", "&").Replace("\"", "").Replace(" ", ""); GramFeed_UserFollower = obj_globul.getHtmlfromUrl_withProxt(new Uri(hit_data), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); if (string.IsNullOrEmpty(GramFeed_UserFollower)) { if (lst_workingfinalproxy.Count > 0) { GramFeed_UserFollower = obj_ChangeProxy.chnageproxyMethod(hit_data); } else { Thread.Sleep(1 * 3000); GramFeed_UserFollower = obj_globul.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); } } string[] follower_regexlist2 = Regex.Split(GramFeed_UserFollower, "username"); foreach (string item in follower_regexlist2) { if (item.Contains("profile_picture")) { string user_name = Utils.getBetween(item, "\":", "\",").Replace("\"", "").Replace(" ", ""); if (follower_list.Count < Number_Follower && !follower_list.Contains(user_name)) { follower_list.Add(user_name); follower_list = follower_list.Distinct().ToList(); ClGlobul.lst_follower_UserFollower.Add(user_name); try { GlobusLogHelper.log.Info(obj_Gram.username + " --> Scraped ===> " + user_name); } catch (Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } } else { break; } } } if (UserFollower_Delaymin != 0) { mindelay = UserFollower_Delaymin; } if (UserFollower_Delaymax != 0) { maxdelay = UserFollower_Delaymax; } delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay); GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds For "); Thread.Sleep(delay * 1000); } else { value = false; } } } // lst_Userfollowerlist = follower_list.ToArray(); ClGlobul.lst_follower_UserFollower = ClGlobul.lst_follower_UserFollower.Distinct().ToList(); lst_Userfollowerlist = ClGlobul.lst_follower_UserFollower.ToArray(); } foreach (string SubFollower in lst_Userfollowerlist) { FollowActivitystart(ref obj_Gram , SubFollower); } } catch(Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } finally { GlobusLogHelper.log.Info("!!! Process Completed !!!"); } }
public string changeproxy(string url) { string responce = string.Empty; List<string> lst_workingfinalproxy = new List<string>(); try { DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy"); DataTable dt = ds.Tables[0]; foreach (DataRow item in dt.Rows) { lst_workingfinalproxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]); } } catch (Exception ex) { } try { GlobusHttpHelper obj_global = new GlobusHttpHelper(); abc: if (lst_workingfinalproxy.Count > 0) { string proxyaddress = string.Empty; string proxyport = string.Empty; string proxyusername = string.Empty; string proxypassword = string.Empty; bool value = true; while (value) { string proxy = lst_workingfinalproxy[RandomNumberGenerator.GenerateRandom(0, lst_workingfinalproxy.Count)]; lst_workingfinalproxy.Remove(proxy); string[] proxydata = Regex.Split(proxy, ":"); if (proxydata.Count() == 2) { proxyaddress = proxydata[0]; proxyport = proxydata[1]; } if (proxydata.Count() == 4) { proxyaddress = proxydata[0]; proxyport = proxydata[1]; proxyusername = proxydata[2]; proxypassword = proxydata[3]; } responce = obj_global.getHtmlfromUrl_withProxt(new Uri(url), proxyaddress, int.Parse(proxyport), proxyusername, proxypassword); if (string.IsNullOrEmpty(responce)) { Thread.Sleep(5 * 1000); goto abc; } else { return responce; } } return responce; } else { GlobusLogHelper.log.Info("Please Upload Working Proxy to Scrape Data"); return null; } return responce; } catch (Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } return responce; }
public void StartScrapeFollowingwithoutlogin() { GlobusHttpHelper obj_GlobusHttpHelper = new GlobusHttpHelper(); // isScrapeFollowing = false; if (isStopScrapeUser) { return; } try { lstofThreadScrapeUser.Add(Thread.CurrentThread); lstofThreadScrapeUser = lstofThreadScrapeUser.Distinct().ToList(); Thread.CurrentThread.IsBackground = true; } catch (Exception ex) { GlobusLogHelper.log.Error("Error : " + ex.StackTrace); } string ProxyAddress = string.Empty; string ProxyPort = "80"; string ProxyUsername = string.Empty; string ProxyPassword = string.Empty; GlobusHttpHelper obj_Following = new GlobusHttpHelper(); List<string> lst_workingfinalproxy = new List<string>(); try { DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy"); DataTable dt = ds.Tables[0]; foreach (DataRow item in dt.Rows) { lst_workingfinalproxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]); } } catch (Exception ex) { } try { if (lst_workingfinalproxy.Count > 0) { abc: string Responce_Following = obj_Following.getHtmlfromUrl(new Uri("http://www.Instagram.com")); string proxy = lst_workingfinalproxy[RandomNumberGenerator.GenerateRandom(0, lst_workingfinalproxy.Count)]; string[] Data_Proxy = Regex.Split(proxy, ":"); if(Data_Proxy.Count()==2) { ProxyAddress = Data_Proxy[0]; ProxyPort = Data_Proxy[1]; } if (Data_Proxy.Count() == 4) { ProxyAddress = Data_Proxy[0]; ProxyPort = Data_Proxy[1]; ProxyUsername = Data_Proxy[2]; ProxyPassword = Data_Proxy[3]; } } } catch(Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } try { string rawUsernameID = string.Empty; string usernamePgSource = string.Empty; bool value = true; foreach (string usernmeToScrape in listOfFollowing) { int count = 0; List<string> ScrapeFollowingUser = new List<string>(); string res_secondURL = obj_Following.getHtmlfromUrl_withProxt(new Uri("https://www.instagram.com"), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword); try { string gramfeed_user_Url = "http://www.gramfeed.com/" + usernmeToScrape; string GramFeed_Userhit = obj_Following.getHtmlfromUrl_withProxt(new Uri(gramfeed_user_Url), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword); string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&"); string Id_Url = "https://api.instagram.com/v1/users/search?q=" + usernmeToScrape + "&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024"; string Gram_Hit_second = obj_Following.getHtmlfromUrl_withProxt(new Uri(Id_Url), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword); string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "\"id\":", "\",").Replace("\"","").Replace(" ",""); string Follower_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/follows?client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024"; string GramFeed_UserFollower = obj_Following.getHtmlfromUrl_withProxt(new Uri(Follower_Url), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword); if(string.IsNullOrEmpty(GramFeed_UserFollower)) { GramFeed_UserFollower = changeproxy(Follower_Url); if(string.IsNullOrEmpty(GramFeed_UserFollower)) { GlobusLogHelper.log.Info("!! All Proxy not getting responce please upload New !!"); return; } } string[] following_regexlist = Regex.Split(GramFeed_UserFollower, "username"); foreach (string var in following_regexlist) { if (var.Contains("profile_picture")) { string user_name = Utils.getBetween(var, "\":", "\",").Replace("\"","").Replace(" ",""); if (ScrapeFollowingUser.Count < noOfUserToScrape && !ScrapeFollowingUser.Contains(user_name)) { ScrapeFollowingUser.Add(user_name); ScrapeFollowingUser = ScrapeFollowingUser.Distinct().ToList(); ClGlobul.lstfollowingList_final.Add(user_name); try { GlobusLogHelper.log.Info("ScrapeUser===> " + user_name + " Form==> " + usernmeToScrape + " With Account " ); string CSVData = "WithoutAccount".Replace(",", string.Empty) + ","+usernmeToScrape.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty); GlobusFileHelper.ExportDataCSVFile(CSVHeader_followingwithoutlogin, CSVData, CSVPath_followingwithoutlogin); } catch (Exception ex) { //GlobusLogHelper.log.Error("Error ==> " + ex.StackTrace); } } else if (ScrapeFollowingUser.Count >= noOfUserToScrape) { break; } } } if (minDelayScrapeUser != 0) { mindelay = minDelayScrapeUser; } if (maxDelayScrapeUser != 0) { maxdelay = maxDelayScrapeUser; } int delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay); GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds "); Thread.Sleep(delay * 1000); if (GramFeed_UserFollower.Contains("next_url")) { value = true; while (value) { if (GramFeed_UserFollower.Contains("next_url")) { string Pagination_UrlFollowing = Utils.getBetween(GramFeed_UserFollower, "next_url\":", "\",").Replace("\\", "").Replace("u0026", "&").Replace(" ","").Replace("\"",""); GramFeed_UserFollower = obj_Following.getHtmlfromUrl_withProxt(new Uri(Pagination_UrlFollowing), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword); if(string.IsNullOrEmpty(GramFeed_UserFollower)) { GramFeed_UserFollower = changeproxy(Pagination_UrlFollowing); } string[] following_regexlist2 = Regex.Split(GramFeed_UserFollower, "username"); foreach (string itemm in following_regexlist2) { if (itemm.Contains("profile_picture")) { string follower = Utils.getBetween(itemm, "\":", "\",").Replace("\"", "").Replace(" ", ""); if (ScrapeFollowingUser.Count < noOfUserToScrape && !ScrapeFollowingUser.Contains(follower) && !ClGlobul.lstfollowingList_final.Contains(follower)) { //if (count == noOfUserPerAccount) //{ // // count = 0; // break; //} ScrapeFollowingUser.Add(follower); ScrapeFollowingUser = ScrapeFollowingUser.Distinct().ToList(); ClGlobul.lstfollowingList_final.Add(follower); try { GlobusLogHelper.log.Info("ScrapeUser===> " + follower + " Form==> " + usernmeToScrape + " With Account " ); string CSVData = "WithoutAccount".Replace(",", string.Empty) + "," + usernmeToScrape.Replace(",", string.Empty) + "," + follower.Replace(",", string.Empty); GlobusFileHelper.ExportDataCSVFile(CSVHeader_followingwithoutlogin, CSVData, CSVPath_followingwithoutlogin); } catch (Exception ex) { // GlobusLogHelper.log.Error("Error ==> " + ex.StackTrace); } } } } if (minDelayScrapeUser != 0) { mindelay = minDelayScrapeUser; } if (maxDelayScrapeUser != 0) { maxdelay = maxDelayScrapeUser; } delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay); GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds "); Thread.Sleep(delay * 1000); } else { value = false; } } } // } } catch (Exception ex) { } } } catch (Exception ex) { // GlobusLogHelper.log.Info("Error :" + ex.StackTrace); } GlobusLogHelper.log.Info("Process Completed" ); }
public void Start_ScrapeCommentuser_WithoutAccout() { int count = 0; List<string> ListofUser_commentOnphoto = new List<string>(); List<string> PhotoId_list = new List<string>(); GlobusHttpHelper Obj_Globosoft = new GlobusHttpHelper(); List<string> lst_WorkingFinalProxy = new List<string>(); string ProxyAd = string.Empty; string ProxyPort = "80"; string ProxyUsername = string.Empty; string ProxyPass = string.Empty; try { DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy"); DataTable dt = ds.Tables[0]; foreach (DataRow item in dt.Rows) { lst_WorkingFinalProxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]); } } catch(Exception ex) { } try { if (lst_WorkingFinalProxy.Count > 0) { string proxy = lst_WorkingFinalProxy[RandomNumberGenerator.GenerateRandom(0, lst_WorkingFinalProxy.Count)]; string[] Data_Proxy = Regex.Split(proxy, ":"); if (Data_Proxy.Count() == 2) { ProxyAd = Data_Proxy[0]; ProxyPort = Data_Proxy[1]; } if (Data_Proxy.Count() == 4) { ProxyAd = Data_Proxy[0]; ProxyPort = Data_Proxy[1]; ProxyUsername = Data_Proxy[2]; ProxyPass = Data_Proxy[3]; } } } catch (Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } try { string username = string.Empty; foreach (string item1 in listOfUsernameForCommentuserScraper) { username = item1; int temp = noOfPhotoToScrape; bool value = true; string Url_user = "******" + username; string responce_user = Obj_Globosoft.getHtmlfromUrl_withProxt(new Uri(Url_user),ProxyAd,int.Parse(ProxyPort),ProxyUsername,ProxyPass); if(string.IsNullOrEmpty(responce_user)) { responce_user = changeproxy(Url_user); } string token = Utils.getBetween(responce_user, "csrf_token\":\"", "\"}"); string[] photo_code = Regex.Split(responce_user, "code\":"); foreach (string list in photo_code) { if (list.Contains("date")) { if (PhotoId_list.Count < temp) { string photo_codes = Utils.getBetween("@" + list, "@\"", "\""); PhotoId_list.Add(photo_codes); } else { break; } } } if (responce_user.Contains("has_next_page\":true")) { while (value) { if (responce_user.Contains("has_next_page\":true") && PhotoId_list.Count < temp) { string IDD = Utils.getBetween(responce_user, "\"id\":\"", "\""); string code_ID = Utils.getBetween(responce_user, "end_cursor\":\"", "\""); string postdata = "q=ig_user(" + IDD + ")+%7B+media.after(" + code_ID + "%2C+12)+%7B%0A++count%2C%0A++nodes+%7B%0A++++caption%2C%0A++++code%2C%0A++++comments+%7B%0A++++++count%0A++++%7D%2C%0A++++date%2C%0A++++dimensions+%7B%0A++++++height%2C%0A++++++width%0A++++%7D%2C%0A++++display_src%2C%0A++++id%2C%0A++++is_video%2C%0A++++likes+%7B%0A++++++count%0A++++%7D%2C%0A++++owner+%7B%0A++++++id%0A++++%7D%2C%0A++++thumbnail_src%0A++%7D%2C%0A++page_info%0A%7D%0A+%7D&ref=users%3A%3Ashow"; Obj_Globosoft.proxyAddress = ProxyAd; Obj_Globosoft.port = int.Parse(ProxyPort); Obj_Globosoft.proxyUsername = ProxyUsername; Obj_Globosoft.proxyPassword = ProxyPass; responce_user = Obj_Globosoft.postFormDatainta(new Uri("https://www.instagram.com/query/"), postdata, "https://www.instagram.com/" + username, token); if(string.IsNullOrEmpty(responce_user)) { responce_user = changeproxy_post("https://www.instagram.com/query/", postdata, "https://www.instagram.com/" + username, token); } string[] data1 = Regex.Split(responce_user, "code"); foreach (string val in data1) { if (val.Contains("date")) { if (PhotoId_list.Count < temp) { string photo_codes = Utils.getBetween(val, "\":\"", "\""); PhotoId_list.Add(photo_codes); string imageUrl = "https://www.instagram.com/p/" + photo_codes + "/"; try { DataBaseHandler.InsertQuery("insert into ScrapedImage(Username,ImageURL,ImageID) values('" + username + "','" + imageUrl + "','" + photo_codes + "')", "ScrapedImage"); } catch (Exception ex) { } } else { break; } } } } else { value = false; } } } } foreach (string item in PhotoId_list) { if (ListofUser_commentOnphoto.Count >= noOfUserToScrape) { break; } try { string url = "https://www.instagram.com/p/" + item + "/"; string responce = Obj_Globosoft.getHtmlfromUrl_withProxt(new Uri(url),ProxyAd,int.Parse(ProxyPort),ProxyUsername,ProxyPass); if (responce.Contains("comments\":{\"count")) { string[] data = Regex.Split(responce, "text"); foreach (string itemm in data) { if (itemm.Contains("username")) { if (ListofUser_commentOnphoto.Count < noOfUserToScrape) { string Username_List = Utils.getBetween(itemm, "username\":\"", "\""); if (!ListofUser_commentOnphoto.Contains(Username_List)) { ListofUser_commentOnphoto.Add(Username_List); ListofUser_commentOnphoto = ListofUser_commentOnphoto.Distinct().ToList(); try { GlobusLogHelper.log.Info("ScrapeUser===>" + Username_List + "Form==>" + username); string CSVData = "WithoutLogin".Replace(",", string.Empty) + "," + username.Replace(",", string.Empty) + "," + url.Replace(",", string.Empty) + "," + Username_List.Replace(",", string.Empty); GlobusFileHelper.ExportDataCSVFile(CVSHeader_PhotoCommentUser, CSVData, CSVPath_PhotoCommentUrl); } catch (Exception ex) { // GlobusLogHelper.log.Error("Error ==> " + ex.Message); } try { DataBaseHandler.InsertQuery("insert into ScrapedUsername(Username) values('" + Username_List + "')", "ScrapedUsername"); count++; } catch (Exception ex) { // GlobusLogHelper.log.Error("Error ==> " + ex.Message); } } else { //break; } } else { break; } //ListofUser_commentOnphoto = ListofUser_commentOnphoto.Distinct().ToList(); } } } } catch (Exception ex) { // GlobusLogHelper.log.Info("Error:" + ex.StackTrace); } } } catch (Exception ex) { // } finally { GlobusLogHelper.log.Info("!! proccess Completed Successfully Scrape Comment User !!"); } }
public void Start_scrapePhotolike_WithoutLogin() { try { int temp = noOfPhotoToScrape; string response = string.Empty; List<string> photoId_list = new List<string>(); string ProxyAd = string.Empty; string ProxyPort = "80"; string ProxyUsername = string.Empty; string ProxyPass = string.Empty; string respon_scrapeuser = string.Empty; bool value = true; List<string> lst_WorkingProxyfinalList = new List<string>(); GlobusHttpHelper obj_global = new GlobusHttpHelper(); try { try { DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy"); DataTable dt = ds.Tables[0]; foreach (DataRow item in dt.Rows) { lst_WorkingProxyfinalList.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]); } } catch (Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } } catch (Exception ex) { GlobusLogHelper.log.Error("Error: " + ex.StackTrace); } try { if (lst_WorkingProxyfinalList.Count > 0) { string proxy = lst_WorkingProxyfinalList[RandomNumberGenerator.GenerateRandom(0, lst_WorkingProxyfinalList.Count)]; string[] Data_Proxy = Regex.Split(proxy, ":"); if (Data_Proxy.Count() == 2) { ProxyAd = Data_Proxy[0]; ProxyPort = Data_Proxy[1]; } if (Data_Proxy.Count() == 4) { ProxyAd = Data_Proxy[0]; ProxyPort = Data_Proxy[1]; ProxyUsername = Data_Proxy[2]; ProxyPass = Data_Proxy[3]; } } } catch (Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } foreach (string username_liker in listOfUsernameForPhotouserScraper) { int count = 0; List<string> User_photolike = new List<string>(); try { string gramfeed_user_Url = "http://www.gramfeed.com/" + username_liker; string GramFeed_Userhit = obj_global.getHtmlfromUrl_withProxt(new Uri(gramfeed_user_Url),ProxyAd,int.Parse(ProxyPort),ProxyUsername,ProxyPass); if(string.IsNullOrEmpty(GramFeed_Userhit)) { GramFeed_Userhit = changeproxy(gramfeed_user_Url); } string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&"); string Id_Url = "https://api.instagram.com/v1/users/search?q=" + username_liker + "&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024"; string Gram_Hit_second = obj_global.getHtmlfromUrl_withProxt(new Uri(Id_Url), ProxyAd, int.Parse(ProxyPort), ProxyUsername, ProxyPass); if(string.IsNullOrEmpty(Gram_Hit_second)) { Gram_Hit_second = changeproxy(Id_Url); } string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "id\":\"", "\""); string Photos_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/media/recent?max_timestamp=&min_timestamp=&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024"; string GramFeed_UserPhotos = obj_global.getHtmlfromUrl_withProxt(new Uri(Photos_Url), ProxyAd, int.Parse(ProxyPort), ProxyUsername, ProxyPass); if(string.IsNullOrEmpty(GramFeed_UserPhotos)) { GramFeed_UserPhotos = changeproxy(Photos_Url); } string[] Photos_regexlist = Regex.Split(GramFeed_UserPhotos, "count"); foreach (string itemmm in Photos_regexlist) { try { if (itemmm.Contains("\"created_time\":\"") && itemmm.Contains("likes")) { if (temp > photoId_list.Count()) { string photo_ID = Utils.getBetween(itemmm, "link\":\"", "\"likes").Replace("\\", ""); photo_ID = Utils.getBetween(photo_ID, "p/", "/"); string imageUrl = "https://www.instagram.com/p/" + photo_ID + "/"; photoId_list.Add(photo_ID); try { DataBaseHandler.InsertQuery("insert into ScrapedImage(Username,ImageURL,ImageID) values('" + username_liker + "','" + imageUrl + "','" + photo_ID + "')", "ScrapedImage"); } catch (Exception ex) { } } } } catch (Exception ex) { // GlobusLogHelper.log.Info("Error:" + ex.StackTrace); } } if (GramFeed_UserPhotos.Contains("next_url")) { while (value) { if (GramFeed_UserPhotos.Contains("next_url") && temp > photoId_list.Count()) { string photo_Url = Utils.getBetween(GramFeed_UserPhotos, "next_url\":\"", "\"").Replace("\\", "").Replace("u0026", "&"); GramFeed_UserPhotos = obj_global.getHtmlfromUrl_withProxt(new Uri(photo_Url),ProxyAd,int.Parse(ProxyPort),ProxyUsername,ProxyPass); if(string.IsNullOrEmpty(GramFeed_UserPhotos)) { GramFeed_UserPhotos = changeproxy(photo_Url); } string[] Photos_regexlist2 = Regex.Split(GramFeed_UserPhotos, "count"); foreach (string itemmm in Photos_regexlist2) { try { if (itemmm.Contains("\"created_time\":\"") && itemmm.Contains("likes")) { if (temp > photoId_list.Count()) { string photo_ID = Utils.getBetween(itemmm, "link\":\"", "\"likes").Replace("\\", ""); photo_ID = Utils.getBetween(photo_ID, "p/", "/"); string imageUrl = "https://www.instagram.com/p/" + photo_ID + "/"; photoId_list.Add(photo_ID); try { DataBaseHandler.InsertQuery("insert into ScrapedImage(Username,ImageURL,ImageID) values('" + username_liker + "','" + imageUrl + "','" + photo_ID + "')", "ScrapedImage"); } catch (Exception ex) { } } } } catch (Exception ex) { // GlobusLogHelper.log.Info("Error:" + ex.StackTrace); } } } else { value = false; } } } } catch (Exception ex) { } try { foreach (string item_Photo in photoId_list) { if (User_photolike.Count == noOfUserToScrape) { break; } try { string Url = "http://www.gramfeed.com/p/" + item_Photo + "/"; string Resp_Url = obj_global.getHtmlfromUrl(new Uri(Url), ""); string client_Id = Utils.getBetween(Resp_Url, "client_id=", "&"); string Image_Id_Url = "https://api.instagram.com/v1/media/shortcode/" + item_Photo + "?client_id=" + client_Id; string Inage_Id_resp = obj_global.getHtmlfromUrl_withProxt(new Uri(Image_Id_Url), ProxyAd, int.Parse(ProxyPort), ProxyUsername, ProxyPass); if(string.IsNullOrEmpty(Inage_Id_resp)) { Inage_Id_resp = changeproxy(Image_Id_Url); } string Id_Image = Utils.getBetween(Inage_Id_resp, "image\",\"id\":\"", "\""); string Like_Url = "https://api.instagram.com/v1/media/" + Id_Image + "/likes?client_id=" + client_Id; string photoliker_list = obj_global.getHtmlfromUrl_withProxt(new Uri(Like_Url), ProxyAd, int.Parse(ProxyPort), ProxyUsername, ProxyPass); if (string.IsNullOrEmpty(photoliker_list)) { photoliker_list = changeproxy(Like_Url); } string[] photolist = Regex.Split(photoliker_list, "username"); foreach (string itemm in photolist) { if (itemm.Contains("profile_picture")) { string Username_photolike = Utils.getBetween(itemm, "\":\"", "\""); if (noOfUserToScrape >= User_photolike.Count && !User_photolike.Contains(Username_photolike)) { User_photolike.Add(Username_photolike); User_photolike = User_photolike.Distinct().ToList(); ClGlobul.lstScrapeLikeuser_final.Add(Username_photolike); try { GlobusLogHelper.log.Info("Scraped User" +" --> " + Username_photolike + "--> From Image Url==>" + Url); string CSVData = "WithoutAccount".Replace(",", string.Empty) + "," + username_liker.Replace(",", string.Empty) + "," + Url.Replace(",", string.Empty) + "," + Username_photolike.Replace(",", string.Empty); GlobusFileHelper.ExportDataCSVFile(CVSHeader_PhotoLikerUser, CSVData, CSVPath_PhotoLikerLikerUser); } catch (Exception ex) { // GlobusLogHelper.log.Error("Error ==> " + ex.Message); } try { count++; DataBaseHandler.InsertQuery("insert into ScrapedUsername(Username) values('" + Username_photolike + "')", "ScrapedUsername"); } catch (Exception ex) { // GlobusLogHelper.log.Error("Error ==> " + ex.Message); } } else { break; } } } } catch (Exception ex) { } } } catch (Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } } } catch(Exception ex) { } finally { GlobusLogHelper.log.Info("!! Process Completed Scrape User Form Photo Liker !!"); } }
public void StartScrapFollowerWithoutLogin() { try { GlobusHttpHelper obj_GlobusHttpHelper = new GlobusHttpHelper(); try { lstThreadsScrapeFollower.Add(Thread.CurrentThread); lstThreadsScrapeFollower.Distinct(); Thread.CurrentThread.IsBackground = true; } catch (Exception ex) { GlobusLogHelper.log.Error("Error : " + ex.StackTrace); } string rawUsernameID = string.Empty; string usernamePgSource = string.Empty; string proxyAddress = string.Empty; string proxyport = "80"; string proxyusername = string.Empty; string proxypasswrd = string.Empty; List<string> lst_workingfinalproxy = new List<string>(); try { DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy"); DataTable dt = ds.Tables[0]; foreach (DataRow item in dt.Rows) { lst_workingfinalproxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]); } } catch(Exception ex) { } try { if (lst_workingfinalproxy.Count > 0) { string res_secondURL = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri("https://www.instagram.com"),proxyAddress,int.Parse(proxyport),proxyusername,proxypasswrd); string proxy = lst_workingfinalproxy[RandomNumberGenerator.GenerateRandom(0, lst_workingfinalproxy.Count)]; string[] proxydata = Regex.Split(proxy, ":"); if(proxydata.Count()==2) { proxyAddress = proxydata[0]; proxyport = proxydata[1]; } if(proxydata.Count()==4) { proxyAddress = proxydata[0]; proxyport = proxydata[1]; proxyusername = proxydata[2]; proxypasswrd = proxydata[3]; } } } catch (Exception ex) { } try { bool val = true; foreach (string itemusername in ClGlobul.listOfScrapeFollowerUserame) { string gramfeed_user_Url = "http://www.gramfeed.com/" + itemusername; Username_ScrapFollower = itemusername; List<string> follower_list = new List<string>(); int count = 0; ClGlobul.switchAccount = false; string GramFeed_Userhit = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(gramfeed_user_Url),proxyAddress,int.Parse(proxyport),proxyusername,proxypasswrd); string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&"); string Id_Url = "https://api.instagram.com/v1/users/search?q=" + itemusername + "&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024"; string Gram_Hit_second = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(Id_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); if(string.IsNullOrEmpty(Gram_Hit_second)) { Thread.Sleep(1 * 3000); Gram_Hit_second = changeproxy(Id_Url); } string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "\"id\":", "\",").Replace("\"","").Replace(" ",""); string Follower_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/followed-by?client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024"; string GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(Follower_Url),proxyAddress,int.Parse(proxyport),proxyusername,proxypasswrd); if (string.IsNullOrEmpty(GramFeed_UserFollower)) { if (lst_workingfinalproxy.Count > 0) { GramFeed_UserFollower = changeproxy(Follower_Url); } else { Thread.Sleep(1 * 3000); GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); } } string[] follower_regexlist = Regex.Split(GramFeed_UserFollower, "username"); foreach (string item in follower_regexlist) { if (item.Contains("profile_picture")) { string user_name = Utils.getBetween(item, "\":", "\",").Replace("\"","").Replace(" ",""); if (follower_list.Count < nOOfUserToScrape && !follower_list.Contains(user_name) && !ClGlobul.lstfollowerList_final.Contains(user_name)) { follower_list.Add(user_name); follower_list = follower_list.Distinct().ToList(); ClGlobul.lstfollowerList_final.Add(user_name); try { GlobusLogHelper.log.Info("Scraped===>" + user_name); string CSVData = "WithoutAccount".Replace(",", string.Empty) + "," + itemusername.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty); GlobusFileHelper.ExportDataCSVFile(CSVHeader_Follower, CSVData, CSVPath_Follower); } catch (Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } try { DataBaseHandler.InsertQuery("insert into ScrapedUsername(Username) values('" + user_name + "')", "ScrapedUsername"); count++; try { DataBaseHandler.InsertQuery("insert into tb_UserDetails(UserName,FriendsId) values('" + "WithoutAccount" + "','" + user_name + "')", "tb_UserDetails"); } catch (Exception ex) { } } catch (Exception ex) { } } else { break; } } } if (Mindelay_ScarpeFollower != 0) { mindelay = Mindelay_ScarpeFollower; } if (Maxdelay_ScarpeFollower != 0) { maxdelay = Maxdelay_ScarpeFollower; } int delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay); GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds "); Thread.Sleep(delay * 1000); if (GramFeed_UserFollower.Contains("next_url")) { value = true; while (value) { if (GramFeed_UserFollower.Contains("next_url") && count < nOOfUserToScrape) { string hit_data = Utils.getBetween(GramFeed_UserFollower, "next_url\":", "\",").Replace("\u0026", "&").Replace("\\", "").Replace("u0026", "&").Replace("\"","").Replace(" ",""); GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(hit_data), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); if (string.IsNullOrEmpty(GramFeed_UserFollower)) { if (lst_workingfinalproxy.Count > 0) { GramFeed_UserFollower = changeproxy(hit_data); } else { Thread.Sleep(1 * 3000); GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd); } } string[] follower_regexlist2 = Regex.Split(GramFeed_UserFollower, "username"); foreach (string item in follower_regexlist2) { if (item.Contains("profile_picture")) { string user_name = Utils.getBetween(item, "\":", "\",").Replace("\"", "").Replace(" ", ""); if (follower_list.Count < nOOfUserToScrape && !follower_list.Contains(user_name) && !ClGlobul.lstfollowerList_final.Contains(user_name)) { follower_list.Add(user_name); follower_list = follower_list.Distinct().ToList(); ClGlobul.lstfollowerList_final.Add(user_name); try { GlobusLogHelper.log.Info("Scraped===>" + user_name); string CSVData = "WithoutAccount".Replace(",", string.Empty) + "," + itemusername.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty); GlobusFileHelper.ExportDataCSVFile(CSVHeader_Follower, CSVData, CSVPath_Follower); } catch (Exception ex) { GlobusLogHelper.log.Error("Error:" + ex.StackTrace); } try { DataBaseHandler.InsertQuery("insert into ScrapedUsername(Username) values('" + user_name + "')", "ScrapedUsername"); count++; try { DataBaseHandler.InsertQuery("insert into tb_UserDetails(UserName,FriendsId) values('" + "WithoutAccount" + "','" + user_name + "')", "tb_UserDetails"); } catch (Exception ex) { } } catch (Exception ex) { } } else { break; } } } } else { value = false; } if (minDelayScrapeUser != 0) { mindelay = minDelayScrapeUser; } if (maxDelayScrapeUser != 0) { maxdelay = maxDelayScrapeUser; } delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay); GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds For "); Thread.Sleep(delay * 1000); } } } //foreach (string itemusername in ClGlobul.listOfScrapeFollowerUserame) //{ // string gramfeed_user_Url = "http://www.gramfeed.com/" + itemusername; // Username_ScrapFollower = itemusername; // List<string> follower_list = new List<string>(); // int count = 0; // ClGlobul.switchAccount = false; // string GramFeed_Userhit = obj_GlobusHttpHelper.getHtmlfromUrl(new Uri(gramfeed_user_Url), ""); // string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&"); // string Id_Url = "https://api.instagram.com/v1/users/search?q=" + itemusername + "&client_id=" + client_Id; // string Gram_Hit_second = obj_GlobusHttpHelper.getHtmlfromUrl(new Uri(Id_Url), ""); // string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "id\":\"", "\""); // string Follower_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/followed-by?client_id=" + client_Id; // string GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl(new Uri(Follower_Url), ""); // string[] follower_regexlist = Regex.Split(GramFeed_UserFollower, "username"); // foreach (string item in follower_regexlist) // { // if (item.Contains("profile_picture")) // { // string user_name = Utils.getBetween(item, "\":\"", "\""); // if (follower_list.Count < nOOfUserToScrape && !follower_list.Contains(user_name)) // { // follower_list.Add(user_name); // follower_list = follower_list.Distinct().ToList(); // ClGlobul.lstfollowerList_final.Add(user_name); // // GlobusLogHelper.log.Info("Scraped===>" + user_name); // try // { // GlobusLogHelper.log.Info("Scraped===>" + user_name); // string CSVData = itemusername.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty); // GlobusFileHelper.ExportDataCSVFile(CSVHeader_Followerwithoutlogin, CSVData, CSVPath_Followerwithoutlogin); // } // catch (Exception ex) // { // GlobusLogHelper.log.Error("Error:" + ex.StackTrace); // } // } // else // { // break; // } // } // } // if (GramFeed_UserFollower.Contains("next_url")) // { // value = true; // while (value) // { // if (GramFeed_UserFollower.Contains("next_url") ) // { // string hit_data = Utils.getBetween(GramFeed_UserFollower, "next_url\":\"", "\",").Replace("\u0026", "&").Replace("\\", "").Replace("u0026", "&"); // GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl(new Uri(hit_data), ""); // string[] follower_regexlist2 = Regex.Split(GramFeed_UserFollower, "username"); // foreach (string item in follower_regexlist2) // { // if (item.Contains("profile_picture")) // { // string user_name = Utils.getBetween(item, "\":\"", "\""); // if (follower_list.Count < nOOfUserToScrape && !follower_list.Contains(user_name)) // { // follower_list.Add(user_name); // follower_list = follower_list.Distinct().ToList(); // ClGlobul.lstfollowerList_final.Add(user_name); // try // { // GlobusLogHelper.log.Info("Scraped===>" + user_name); // string CSVData = itemusername.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty); // GlobusFileHelper.ExportDataCSVFile(CSVHeader_Followerwithoutlogin, CSVData, CSVPath_Followerwithoutlogin); // } // catch (Exception ex) // { // GlobusLogHelper.log.Error("Error:" + ex.StackTrace); // } // } // else // { // break; // } // } // } // } // else // { // value = false; // } // } // } //} } catch (Exception ex) { } GlobusLogHelper.log.Info(" Process Completed By "); } catch(Exception ex) { } }