public void Start_Action_UserFollower(ref InstagramUser obj_Gram)
{
try
{
lstThreadsUserFollower.Add(Thread.CurrentThread);
lstThreadsUserFollower.Distinct();
Thread.CurrentThread.IsBackground = true;
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error : " + ex.StackTrace);
}
try
{
string[] lst_UserFollower_listedUser = ClGlobul.lstUserFollower_Username.ToArray();
GlobusHttpHelper obj_globul = new GlobusHttpHelper();
bool value = false;
string proxyAddress = string.Empty;
string proxyport = "80";
string proxyusername = string.Empty;
string proxypasswrd = string.Empty;
List<string> lst_workingfinalproxy = new List<string>();
string[] lst_Userfollowerlist = {};
try
{
DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy");
DataTable dt = ds.Tables[0];
foreach (DataRow item in dt.Rows)
{
lst_workingfinalproxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]);
}
}
catch (Exception ex)
{
}
try
{
if (lst_workingfinalproxy.Count > 0)
{
string res_secondURL = obj_Gram.globusHttpHelper.getHtmlfromUrl_withProxt(new Uri("https://www.instagram.com"), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
string proxy = lst_workingfinalproxy[RandomNumberGenerator.GenerateRandom(0, lst_workingfinalproxy.Count)];
string[] proxydata = Regex.Split(proxy, ":");
if (proxydata.Count() == 2)
{
proxyAddress = proxydata[0];
proxyport = proxydata[1];
}
if (proxydata.Count() == 4)
{
proxyAddress = proxydata[0];
proxyport = proxydata[1];
proxyusername = proxydata[2];
proxypasswrd = proxydata[3];
}
}
}
catch (Exception ex)
{
}
foreach(string username in lst_UserFollower_listedUser)
{
string gramfeed_user_Url = "http://www.gramfeed.com/" + username;
// Username_ScrapFollower = itemusername;
List<string> follower_list = new List<string>();
int count = 0;
ClGlobul.switchAccount = false;
string GramFeed_Userhit = obj_globul.getHtmlfromUrl_withProxt(new Uri(gramfeed_user_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&");
string Id_Url = "https://api.instagram.com/v1/users/search?q=" + username + "&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024";
string Gram_Hit_second = obj_globul.getHtmlfromUrl_withProxt(new Uri(Id_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
if (string.IsNullOrEmpty(Gram_Hit_second))
{
Thread.Sleep(1 * 3000);
Gram_Hit_second = obj_ChangeProxy.chnageproxyMethod(Id_Url);
}
string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "\"id\":", "\",").Replace("\"", "").Replace(" ", "");
string Follower_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/followed-by?client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024";
string GramFeed_UserFollower = obj_globul.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
if (string.IsNullOrEmpty(GramFeed_UserFollower))
{
if (lst_workingfinalproxy.Count > 0)
{
GramFeed_UserFollower = obj_ChangeProxy.chnageproxyMethod(Follower_Url);
}
else
{
Thread.Sleep(1 * 3000);
GramFeed_UserFollower = obj_globul.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
}
}
string[] follower_regexlist = Regex.Split(GramFeed_UserFollower, "username");
foreach (string item in follower_regexlist)
{
if (item.Contains("profile_picture"))
{
string user_name = Utils.getBetween(item, "\":", "\",").Replace("\"", "").Replace(" ", "");
if (follower_list.Count < Number_Follower && !follower_list.Contains(user_name))
{
follower_list.Add(user_name);
follower_list = follower_list.Distinct().ToList();
ClGlobul.lst_follower_UserFollower.Add(user_name);
count++;
try
{
GlobusLogHelper.log.Info(obj_Gram.username +" --> Scraped ===> " + user_name);
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
}
else
{
break;
}
}
}
if (UserFollower_Delaymin != 0)
{
mindelay = UserFollower_Delaymin;
}
if (UserFollower_Delaymax != 0)
{
maxdelay = UserFollower_Delaymax;
}
int delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay);
GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds ");
Thread.Sleep(delay * 1000);
if (GramFeed_UserFollower.Contains("next_url"))
{
value = true;
while (value)
{
if (GramFeed_UserFollower.Contains("next_url") && follower_list.Count < Number_Follower)
{
string hit_data = Utils.getBetween(GramFeed_UserFollower, "next_url\":", "\",").Replace("\u0026", "&").Replace("\\", "").Replace("u0026", "&").Replace("\"", "").Replace(" ", "");
GramFeed_UserFollower = obj_globul.getHtmlfromUrl_withProxt(new Uri(hit_data), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
if (string.IsNullOrEmpty(GramFeed_UserFollower))
{
if (lst_workingfinalproxy.Count > 0)
{
GramFeed_UserFollower = obj_ChangeProxy.chnageproxyMethod(hit_data);
}
else
{
Thread.Sleep(1 * 3000);
GramFeed_UserFollower = obj_globul.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
}
}
string[] follower_regexlist2 = Regex.Split(GramFeed_UserFollower, "username");
foreach (string item in follower_regexlist2)
{
if (item.Contains("profile_picture"))
{
string user_name = Utils.getBetween(item, "\":", "\",").Replace("\"", "").Replace(" ", "");
if (follower_list.Count < Number_Follower && !follower_list.Contains(user_name))
{
follower_list.Add(user_name);
follower_list = follower_list.Distinct().ToList();
ClGlobul.lst_follower_UserFollower.Add(user_name);
try
{
GlobusLogHelper.log.Info(obj_Gram.username + " --> Scraped ===> " + user_name);
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
}
else
{
break;
}
}
}
if (UserFollower_Delaymin != 0)
{
mindelay = UserFollower_Delaymin;
}
if (UserFollower_Delaymax != 0)
{
maxdelay = UserFollower_Delaymax;
}
delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay);
GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds For ");
Thread.Sleep(delay * 1000);
}
else
{
value = false;
}
}
}
// lst_Userfollowerlist = follower_list.ToArray();
ClGlobul.lst_follower_UserFollower = ClGlobul.lst_follower_UserFollower.Distinct().ToList();
lst_Userfollowerlist = ClGlobul.lst_follower_UserFollower.ToArray();
}
foreach (string SubFollower in lst_Userfollowerlist)
{
FollowActivitystart(ref obj_Gram , SubFollower);
}
}
catch(Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
finally
{
GlobusLogHelper.log.Info("!!! Process Completed !!!");
}
}
public string changeproxy(string url)
{
string responce = string.Empty;
List<string> lst_workingfinalproxy = new List<string>();
try
{
DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy");
DataTable dt = ds.Tables[0];
foreach (DataRow item in dt.Rows)
{
lst_workingfinalproxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]);
}
}
catch (Exception ex)
{
}
try
{
GlobusHttpHelper obj_global = new GlobusHttpHelper();
abc:
if (lst_workingfinalproxy.Count > 0)
{
string proxyaddress = string.Empty;
string proxyport = string.Empty;
string proxyusername = string.Empty;
string proxypassword = string.Empty;
bool value = true;
while (value)
{
string proxy = lst_workingfinalproxy[RandomNumberGenerator.GenerateRandom(0, lst_workingfinalproxy.Count)];
lst_workingfinalproxy.Remove(proxy);
string[] proxydata = Regex.Split(proxy, ":");
if (proxydata.Count() == 2)
{
proxyaddress = proxydata[0];
proxyport = proxydata[1];
}
if (proxydata.Count() == 4)
{
proxyaddress = proxydata[0];
proxyport = proxydata[1];
proxyusername = proxydata[2];
proxypassword = proxydata[3];
}
responce = obj_global.getHtmlfromUrl_withProxt(new Uri(url), proxyaddress, int.Parse(proxyport), proxyusername, proxypassword);
if (string.IsNullOrEmpty(responce))
{
Thread.Sleep(5 * 1000);
goto abc;
}
else
{
return responce;
}
}
return responce;
}
else
{
GlobusLogHelper.log.Info("Please Upload Working Proxy to Scrape Data");
return null;
}
return responce;
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
return responce;
}
public void StartScrapeFollowingwithoutlogin()
{
GlobusHttpHelper obj_GlobusHttpHelper = new GlobusHttpHelper();
// isScrapeFollowing = false;
if (isStopScrapeUser)
{
return;
}
try
{
lstofThreadScrapeUser.Add(Thread.CurrentThread);
lstofThreadScrapeUser = lstofThreadScrapeUser.Distinct().ToList();
Thread.CurrentThread.IsBackground = true;
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error : " + ex.StackTrace);
}
string ProxyAddress = string.Empty;
string ProxyPort = "80";
string ProxyUsername = string.Empty;
string ProxyPassword = string.Empty;
GlobusHttpHelper obj_Following = new GlobusHttpHelper();
List<string> lst_workingfinalproxy = new List<string>();
try
{
DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy");
DataTable dt = ds.Tables[0];
foreach (DataRow item in dt.Rows)
{
lst_workingfinalproxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]);
}
}
catch (Exception ex)
{
}
try
{
if (lst_workingfinalproxy.Count > 0)
{
abc:
string Responce_Following = obj_Following.getHtmlfromUrl(new Uri("http://www.Instagram.com"));
string proxy = lst_workingfinalproxy[RandomNumberGenerator.GenerateRandom(0, lst_workingfinalproxy.Count)];
string[] Data_Proxy = Regex.Split(proxy, ":");
if(Data_Proxy.Count()==2)
{
ProxyAddress = Data_Proxy[0];
ProxyPort = Data_Proxy[1];
}
if (Data_Proxy.Count() == 4)
{
ProxyAddress = Data_Proxy[0];
ProxyPort = Data_Proxy[1];
ProxyUsername = Data_Proxy[2];
ProxyPassword = Data_Proxy[3];
}
}
}
catch(Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
try
{
string rawUsernameID = string.Empty;
string usernamePgSource = string.Empty;
bool value = true;
foreach (string usernmeToScrape in listOfFollowing)
{
int count = 0;
List<string> ScrapeFollowingUser = new List<string>();
string res_secondURL = obj_Following.getHtmlfromUrl_withProxt(new Uri("https://www.instagram.com"), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword);
try
{
string gramfeed_user_Url = "http://www.gramfeed.com/" + usernmeToScrape;
string GramFeed_Userhit = obj_Following.getHtmlfromUrl_withProxt(new Uri(gramfeed_user_Url), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword);
string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&");
string Id_Url = "https://api.instagram.com/v1/users/search?q=" + usernmeToScrape + "&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024";
string Gram_Hit_second = obj_Following.getHtmlfromUrl_withProxt(new Uri(Id_Url), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword);
string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "\"id\":", "\",").Replace("\"","").Replace(" ","");
string Follower_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/follows?client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024";
string GramFeed_UserFollower = obj_Following.getHtmlfromUrl_withProxt(new Uri(Follower_Url), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword);
if(string.IsNullOrEmpty(GramFeed_UserFollower))
{
GramFeed_UserFollower = changeproxy(Follower_Url);
if(string.IsNullOrEmpty(GramFeed_UserFollower))
{
GlobusLogHelper.log.Info("!! All Proxy not getting responce please upload New !!");
return;
}
}
string[] following_regexlist = Regex.Split(GramFeed_UserFollower, "username");
foreach (string var in following_regexlist)
{
if (var.Contains("profile_picture"))
{
string user_name = Utils.getBetween(var, "\":", "\",").Replace("\"","").Replace(" ","");
if (ScrapeFollowingUser.Count < noOfUserToScrape && !ScrapeFollowingUser.Contains(user_name))
{
ScrapeFollowingUser.Add(user_name);
ScrapeFollowingUser = ScrapeFollowingUser.Distinct().ToList();
ClGlobul.lstfollowingList_final.Add(user_name);
try
{
GlobusLogHelper.log.Info("ScrapeUser===> " + user_name + " Form==> " + usernmeToScrape + " With Account " );
string CSVData = "WithoutAccount".Replace(",", string.Empty) + ","+usernmeToScrape.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty);
GlobusFileHelper.ExportDataCSVFile(CSVHeader_followingwithoutlogin, CSVData, CSVPath_followingwithoutlogin);
}
catch (Exception ex)
{
//GlobusLogHelper.log.Error("Error ==> " + ex.StackTrace);
}
}
else if (ScrapeFollowingUser.Count >= noOfUserToScrape)
{
break;
}
}
}
if (minDelayScrapeUser != 0)
{
mindelay = minDelayScrapeUser;
}
if (maxDelayScrapeUser != 0)
{
maxdelay = maxDelayScrapeUser;
}
int delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay);
GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds ");
Thread.Sleep(delay * 1000);
if (GramFeed_UserFollower.Contains("next_url"))
{
value = true;
while (value)
{
if (GramFeed_UserFollower.Contains("next_url"))
{
string Pagination_UrlFollowing = Utils.getBetween(GramFeed_UserFollower, "next_url\":", "\",").Replace("\\", "").Replace("u0026", "&").Replace(" ","").Replace("\"","");
GramFeed_UserFollower = obj_Following.getHtmlfromUrl_withProxt(new Uri(Pagination_UrlFollowing), ProxyAddress, int.Parse(ProxyPort), ProxyAddress, ProxyPassword);
if(string.IsNullOrEmpty(GramFeed_UserFollower))
{
GramFeed_UserFollower = changeproxy(Pagination_UrlFollowing);
}
string[] following_regexlist2 = Regex.Split(GramFeed_UserFollower, "username");
foreach (string itemm in following_regexlist2)
{
if (itemm.Contains("profile_picture"))
{
string follower = Utils.getBetween(itemm, "\":", "\",").Replace("\"", "").Replace(" ", "");
if (ScrapeFollowingUser.Count < noOfUserToScrape && !ScrapeFollowingUser.Contains(follower) && !ClGlobul.lstfollowingList_final.Contains(follower))
{
//if (count == noOfUserPerAccount)
//{
// // count = 0;
// break;
//}
ScrapeFollowingUser.Add(follower);
ScrapeFollowingUser = ScrapeFollowingUser.Distinct().ToList();
ClGlobul.lstfollowingList_final.Add(follower);
try
{
GlobusLogHelper.log.Info("ScrapeUser===> " + follower + " Form==> " + usernmeToScrape + " With Account " );
string CSVData = "WithoutAccount".Replace(",", string.Empty) + "," + usernmeToScrape.Replace(",", string.Empty) + "," + follower.Replace(",", string.Empty);
GlobusFileHelper.ExportDataCSVFile(CSVHeader_followingwithoutlogin, CSVData, CSVPath_followingwithoutlogin);
}
catch (Exception ex)
{
// GlobusLogHelper.log.Error("Error ==> " + ex.StackTrace);
}
}
}
}
if (minDelayScrapeUser != 0)
{
mindelay = minDelayScrapeUser;
}
if (maxDelayScrapeUser != 0)
{
maxdelay = maxDelayScrapeUser;
}
delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay);
GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds ");
Thread.Sleep(delay * 1000);
}
else
{
value = false;
}
}
}
// }
}
catch (Exception ex)
{
}
}
}
catch (Exception ex)
{
// GlobusLogHelper.log.Info("Error :" + ex.StackTrace);
}
GlobusLogHelper.log.Info("Process Completed" );
}
public void Start_ScrapeCommentuser_WithoutAccout()
{
int count = 0;
List<string> ListofUser_commentOnphoto = new List<string>();
List<string> PhotoId_list = new List<string>();
GlobusHttpHelper Obj_Globosoft = new GlobusHttpHelper();
List<string> lst_WorkingFinalProxy = new List<string>();
string ProxyAd = string.Empty;
string ProxyPort = "80";
string ProxyUsername = string.Empty;
string ProxyPass = string.Empty;
try
{
DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy");
DataTable dt = ds.Tables[0];
foreach (DataRow item in dt.Rows)
{
lst_WorkingFinalProxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]);
}
}
catch(Exception ex)
{
}
try
{
if (lst_WorkingFinalProxy.Count > 0)
{
string proxy = lst_WorkingFinalProxy[RandomNumberGenerator.GenerateRandom(0, lst_WorkingFinalProxy.Count)];
string[] Data_Proxy = Regex.Split(proxy, ":");
if (Data_Proxy.Count() == 2)
{
ProxyAd = Data_Proxy[0];
ProxyPort = Data_Proxy[1];
}
if (Data_Proxy.Count() == 4)
{
ProxyAd = Data_Proxy[0];
ProxyPort = Data_Proxy[1];
ProxyUsername = Data_Proxy[2];
ProxyPass = Data_Proxy[3];
}
}
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
try
{
string username = string.Empty;
foreach (string item1 in listOfUsernameForCommentuserScraper)
{
username = item1;
int temp = noOfPhotoToScrape;
bool value = true;
string Url_user = "******" + username;
string responce_user = Obj_Globosoft.getHtmlfromUrl_withProxt(new Uri(Url_user),ProxyAd,int.Parse(ProxyPort),ProxyUsername,ProxyPass);
if(string.IsNullOrEmpty(responce_user))
{
responce_user = changeproxy(Url_user);
}
string token = Utils.getBetween(responce_user, "csrf_token\":\"", "\"}");
string[] photo_code = Regex.Split(responce_user, "code\":");
foreach (string list in photo_code)
{
if (list.Contains("date"))
{
if (PhotoId_list.Count < temp)
{
string photo_codes = Utils.getBetween("@" + list, "@\"", "\"");
PhotoId_list.Add(photo_codes);
}
else
{
break;
}
}
}
if (responce_user.Contains("has_next_page\":true"))
{
while (value)
{
if (responce_user.Contains("has_next_page\":true") && PhotoId_list.Count < temp)
{
string IDD = Utils.getBetween(responce_user, "\"id\":\"", "\"");
string code_ID = Utils.getBetween(responce_user, "end_cursor\":\"", "\"");
string postdata = "q=ig_user(" + IDD + ")+%7B+media.after(" + code_ID + "%2C+12)+%7B%0A++count%2C%0A++nodes+%7B%0A++++caption%2C%0A++++code%2C%0A++++comments+%7B%0A++++++count%0A++++%7D%2C%0A++++date%2C%0A++++dimensions+%7B%0A++++++height%2C%0A++++++width%0A++++%7D%2C%0A++++display_src%2C%0A++++id%2C%0A++++is_video%2C%0A++++likes+%7B%0A++++++count%0A++++%7D%2C%0A++++owner+%7B%0A++++++id%0A++++%7D%2C%0A++++thumbnail_src%0A++%7D%2C%0A++page_info%0A%7D%0A+%7D&ref=users%3A%3Ashow";
Obj_Globosoft.proxyAddress = ProxyAd;
Obj_Globosoft.port = int.Parse(ProxyPort);
Obj_Globosoft.proxyUsername = ProxyUsername;
Obj_Globosoft.proxyPassword = ProxyPass;
responce_user = Obj_Globosoft.postFormDatainta(new Uri("https://www.instagram.com/query/"), postdata, "https://www.instagram.com/" + username, token);
if(string.IsNullOrEmpty(responce_user))
{
responce_user = changeproxy_post("https://www.instagram.com/query/", postdata, "https://www.instagram.com/" + username, token);
}
string[] data1 = Regex.Split(responce_user, "code");
foreach (string val in data1)
{
if (val.Contains("date"))
{
if (PhotoId_list.Count < temp)
{
string photo_codes = Utils.getBetween(val, "\":\"", "\"");
PhotoId_list.Add(photo_codes);
string imageUrl = "https://www.instagram.com/p/" + photo_codes + "/";
try
{
DataBaseHandler.InsertQuery("insert into ScrapedImage(Username,ImageURL,ImageID) values('" + username + "','" + imageUrl + "','" + photo_codes + "')", "ScrapedImage");
}
catch (Exception ex)
{
}
}
else
{
break;
}
}
}
}
else
{
value = false;
}
}
}
}
foreach (string item in PhotoId_list)
{
if (ListofUser_commentOnphoto.Count >= noOfUserToScrape)
{
break;
}
try
{
string url = "https://www.instagram.com/p/" + item + "/";
string responce = Obj_Globosoft.getHtmlfromUrl_withProxt(new Uri(url),ProxyAd,int.Parse(ProxyPort),ProxyUsername,ProxyPass);
if (responce.Contains("comments\":{\"count"))
{
string[] data = Regex.Split(responce, "text");
foreach (string itemm in data)
{
if (itemm.Contains("username"))
{
if (ListofUser_commentOnphoto.Count < noOfUserToScrape)
{
string Username_List = Utils.getBetween(itemm, "username\":\"", "\"");
if (!ListofUser_commentOnphoto.Contains(Username_List))
{
ListofUser_commentOnphoto.Add(Username_List);
ListofUser_commentOnphoto = ListofUser_commentOnphoto.Distinct().ToList();
try
{
GlobusLogHelper.log.Info("ScrapeUser===>" + Username_List + "Form==>" + username);
string CSVData = "WithoutLogin".Replace(",", string.Empty) + "," + username.Replace(",", string.Empty) + "," + url.Replace(",", string.Empty) + "," + Username_List.Replace(",", string.Empty);
GlobusFileHelper.ExportDataCSVFile(CVSHeader_PhotoCommentUser, CSVData, CSVPath_PhotoCommentUrl);
}
catch (Exception ex)
{
// GlobusLogHelper.log.Error("Error ==> " + ex.Message);
}
try
{
DataBaseHandler.InsertQuery("insert into ScrapedUsername(Username) values('" + Username_List + "')", "ScrapedUsername");
count++;
}
catch (Exception ex)
{
// GlobusLogHelper.log.Error("Error ==> " + ex.Message);
}
}
else
{
//break;
}
}
else
{
break;
}
//ListofUser_commentOnphoto = ListofUser_commentOnphoto.Distinct().ToList();
}
}
}
}
catch (Exception ex)
{
// GlobusLogHelper.log.Info("Error:" + ex.StackTrace);
}
}
}
catch (Exception ex)
{
//
}
finally
{
GlobusLogHelper.log.Info("!! proccess Completed Successfully Scrape Comment User !!");
}
}
public void Start_scrapePhotolike_WithoutLogin()
{
try
{
int temp = noOfPhotoToScrape;
string response = string.Empty;
List<string> photoId_list = new List<string>();
string ProxyAd = string.Empty;
string ProxyPort = "80";
string ProxyUsername = string.Empty;
string ProxyPass = string.Empty;
string respon_scrapeuser = string.Empty;
bool value = true;
List<string> lst_WorkingProxyfinalList = new List<string>();
GlobusHttpHelper obj_global = new GlobusHttpHelper();
try
{
try
{
DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy");
DataTable dt = ds.Tables[0];
foreach (DataRow item in dt.Rows)
{
lst_WorkingProxyfinalList.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]);
}
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error: " + ex.StackTrace);
}
try
{
if (lst_WorkingProxyfinalList.Count > 0)
{
string proxy = lst_WorkingProxyfinalList[RandomNumberGenerator.GenerateRandom(0, lst_WorkingProxyfinalList.Count)];
string[] Data_Proxy = Regex.Split(proxy, ":");
if (Data_Proxy.Count() == 2)
{
ProxyAd = Data_Proxy[0];
ProxyPort = Data_Proxy[1];
}
if (Data_Proxy.Count() == 4)
{
ProxyAd = Data_Proxy[0];
ProxyPort = Data_Proxy[1];
ProxyUsername = Data_Proxy[2];
ProxyPass = Data_Proxy[3];
}
}
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
foreach (string username_liker in listOfUsernameForPhotouserScraper)
{
int count = 0;
List<string> User_photolike = new List<string>();
try
{
string gramfeed_user_Url = "http://www.gramfeed.com/" + username_liker;
string GramFeed_Userhit = obj_global.getHtmlfromUrl_withProxt(new Uri(gramfeed_user_Url),ProxyAd,int.Parse(ProxyPort),ProxyUsername,ProxyPass);
if(string.IsNullOrEmpty(GramFeed_Userhit))
{
GramFeed_Userhit = changeproxy(gramfeed_user_Url);
}
string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&");
string Id_Url = "https://api.instagram.com/v1/users/search?q=" + username_liker + "&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024";
string Gram_Hit_second = obj_global.getHtmlfromUrl_withProxt(new Uri(Id_Url), ProxyAd, int.Parse(ProxyPort), ProxyUsername, ProxyPass);
if(string.IsNullOrEmpty(Gram_Hit_second))
{
Gram_Hit_second = changeproxy(Id_Url);
}
string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "id\":\"", "\"");
string Photos_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/media/recent?max_timestamp=&min_timestamp=&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024";
string GramFeed_UserPhotos = obj_global.getHtmlfromUrl_withProxt(new Uri(Photos_Url), ProxyAd, int.Parse(ProxyPort), ProxyUsername, ProxyPass);
if(string.IsNullOrEmpty(GramFeed_UserPhotos))
{
GramFeed_UserPhotos = changeproxy(Photos_Url);
}
string[] Photos_regexlist = Regex.Split(GramFeed_UserPhotos, "count");
foreach (string itemmm in Photos_regexlist)
{
try
{
if (itemmm.Contains("\"created_time\":\"") && itemmm.Contains("likes"))
{
if (temp > photoId_list.Count())
{
string photo_ID = Utils.getBetween(itemmm, "link\":\"", "\"likes").Replace("\\", "");
photo_ID = Utils.getBetween(photo_ID, "p/", "/");
string imageUrl = "https://www.instagram.com/p/" + photo_ID + "/";
photoId_list.Add(photo_ID);
try
{
DataBaseHandler.InsertQuery("insert into ScrapedImage(Username,ImageURL,ImageID) values('" + username_liker + "','" + imageUrl + "','" + photo_ID + "')", "ScrapedImage");
}
catch (Exception ex)
{
}
}
}
}
catch (Exception ex)
{
// GlobusLogHelper.log.Info("Error:" + ex.StackTrace);
}
}
if (GramFeed_UserPhotos.Contains("next_url"))
{
while (value)
{
if (GramFeed_UserPhotos.Contains("next_url") && temp > photoId_list.Count())
{
string photo_Url = Utils.getBetween(GramFeed_UserPhotos, "next_url\":\"", "\"").Replace("\\", "").Replace("u0026", "&");
GramFeed_UserPhotos = obj_global.getHtmlfromUrl_withProxt(new Uri(photo_Url),ProxyAd,int.Parse(ProxyPort),ProxyUsername,ProxyPass);
if(string.IsNullOrEmpty(GramFeed_UserPhotos))
{
GramFeed_UserPhotos = changeproxy(photo_Url);
}
string[] Photos_regexlist2 = Regex.Split(GramFeed_UserPhotos, "count");
foreach (string itemmm in Photos_regexlist2)
{
try
{
if (itemmm.Contains("\"created_time\":\"") && itemmm.Contains("likes"))
{
if (temp > photoId_list.Count())
{
string photo_ID = Utils.getBetween(itemmm, "link\":\"", "\"likes").Replace("\\", "");
photo_ID = Utils.getBetween(photo_ID, "p/", "/");
string imageUrl = "https://www.instagram.com/p/" + photo_ID + "/";
photoId_list.Add(photo_ID);
try
{
DataBaseHandler.InsertQuery("insert into ScrapedImage(Username,ImageURL,ImageID) values('" + username_liker + "','" + imageUrl + "','" + photo_ID + "')", "ScrapedImage");
}
catch (Exception ex)
{
}
}
}
}
catch (Exception ex)
{
// GlobusLogHelper.log.Info("Error:" + ex.StackTrace);
}
}
}
else
{
value = false;
}
}
}
}
catch (Exception ex)
{
}
try
{
foreach (string item_Photo in photoId_list)
{
if (User_photolike.Count == noOfUserToScrape)
{
break;
}
try
{
string Url = "http://www.gramfeed.com/p/" + item_Photo + "/";
string Resp_Url = obj_global.getHtmlfromUrl(new Uri(Url), "");
string client_Id = Utils.getBetween(Resp_Url, "client_id=", "&");
string Image_Id_Url = "https://api.instagram.com/v1/media/shortcode/" + item_Photo + "?client_id=" + client_Id;
string Inage_Id_resp = obj_global.getHtmlfromUrl_withProxt(new Uri(Image_Id_Url), ProxyAd, int.Parse(ProxyPort), ProxyUsername, ProxyPass);
if(string.IsNullOrEmpty(Inage_Id_resp))
{
Inage_Id_resp = changeproxy(Image_Id_Url);
}
string Id_Image = Utils.getBetween(Inage_Id_resp, "image\",\"id\":\"", "\"");
string Like_Url = "https://api.instagram.com/v1/media/" + Id_Image + "/likes?client_id=" + client_Id;
string photoliker_list = obj_global.getHtmlfromUrl_withProxt(new Uri(Like_Url), ProxyAd, int.Parse(ProxyPort), ProxyUsername, ProxyPass);
if (string.IsNullOrEmpty(photoliker_list))
{
photoliker_list = changeproxy(Like_Url);
}
string[] photolist = Regex.Split(photoliker_list, "username");
foreach (string itemm in photolist)
{
if (itemm.Contains("profile_picture"))
{
string Username_photolike = Utils.getBetween(itemm, "\":\"", "\"");
if (noOfUserToScrape >= User_photolike.Count && !User_photolike.Contains(Username_photolike))
{
User_photolike.Add(Username_photolike);
User_photolike = User_photolike.Distinct().ToList();
ClGlobul.lstScrapeLikeuser_final.Add(Username_photolike);
try
{
GlobusLogHelper.log.Info("Scraped User" +" --> " + Username_photolike + "--> From Image Url==>" + Url);
string CSVData = "WithoutAccount".Replace(",", string.Empty) + "," + username_liker.Replace(",", string.Empty) + "," + Url.Replace(",", string.Empty) + "," + Username_photolike.Replace(",", string.Empty);
GlobusFileHelper.ExportDataCSVFile(CVSHeader_PhotoLikerUser, CSVData, CSVPath_PhotoLikerLikerUser);
}
catch (Exception ex)
{
// GlobusLogHelper.log.Error("Error ==> " + ex.Message);
}
try
{
count++;
DataBaseHandler.InsertQuery("insert into ScrapedUsername(Username) values('" + Username_photolike + "')", "ScrapedUsername");
}
catch (Exception ex)
{
// GlobusLogHelper.log.Error("Error ==> " + ex.Message);
}
}
else
{
break;
}
}
}
}
catch (Exception ex)
{
}
}
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
}
}
catch(Exception ex)
{
}
finally
{
GlobusLogHelper.log.Info("!! Process Completed Scrape User Form Photo Liker !!");
}
}
public void StartScrapFollowerWithoutLogin()
{
try
{
GlobusHttpHelper obj_GlobusHttpHelper = new GlobusHttpHelper();
try
{
lstThreadsScrapeFollower.Add(Thread.CurrentThread);
lstThreadsScrapeFollower.Distinct();
Thread.CurrentThread.IsBackground = true;
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error : " + ex.StackTrace);
}
string rawUsernameID = string.Empty;
string usernamePgSource = string.Empty;
string proxyAddress = string.Empty;
string proxyport = "80";
string proxyusername = string.Empty;
string proxypasswrd = string.Empty;
List<string> lst_workingfinalproxy = new List<string>();
try
{
DataSet ds = DataBaseHandler.SelectQuery("select * from tlb_WorkingProxy", "tlb_WorkingProxy");
DataTable dt = ds.Tables[0];
foreach (DataRow item in dt.Rows)
{
lst_workingfinalproxy.Add(item[1] + ":" + item[2] + ":" + item[3] + ":" + item[4]);
}
}
catch(Exception ex)
{
}
try
{
if (lst_workingfinalproxy.Count > 0)
{
string res_secondURL = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri("https://www.instagram.com"),proxyAddress,int.Parse(proxyport),proxyusername,proxypasswrd);
string proxy = lst_workingfinalproxy[RandomNumberGenerator.GenerateRandom(0, lst_workingfinalproxy.Count)];
string[] proxydata = Regex.Split(proxy, ":");
if(proxydata.Count()==2)
{
proxyAddress = proxydata[0];
proxyport = proxydata[1];
}
if(proxydata.Count()==4)
{
proxyAddress = proxydata[0];
proxyport = proxydata[1];
proxyusername = proxydata[2];
proxypasswrd = proxydata[3];
}
}
}
catch (Exception ex)
{
}
try
{
bool val = true;
foreach (string itemusername in ClGlobul.listOfScrapeFollowerUserame)
{
string gramfeed_user_Url = "http://www.gramfeed.com/" + itemusername;
Username_ScrapFollower = itemusername;
List<string> follower_list = new List<string>();
int count = 0;
ClGlobul.switchAccount = false;
string GramFeed_Userhit = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(gramfeed_user_Url),proxyAddress,int.Parse(proxyport),proxyusername,proxypasswrd);
string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&");
string Id_Url = "https://api.instagram.com/v1/users/search?q=" + itemusername + "&client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024";
string Gram_Hit_second = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(Id_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
if(string.IsNullOrEmpty(Gram_Hit_second))
{
Thread.Sleep(1 * 3000);
Gram_Hit_second = changeproxy(Id_Url);
}
string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "\"id\":", "\",").Replace("\"","").Replace(" ","");
string Follower_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/followed-by?client_id=" + client_Id + "&callback=jQuery183011914858664385974_1455619732855&_=1455619735024";
string GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(Follower_Url),proxyAddress,int.Parse(proxyport),proxyusername,proxypasswrd);
if (string.IsNullOrEmpty(GramFeed_UserFollower))
{
if (lst_workingfinalproxy.Count > 0)
{
GramFeed_UserFollower = changeproxy(Follower_Url);
}
else
{
Thread.Sleep(1 * 3000);
GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
}
}
string[] follower_regexlist = Regex.Split(GramFeed_UserFollower, "username");
foreach (string item in follower_regexlist)
{
if (item.Contains("profile_picture"))
{
string user_name = Utils.getBetween(item, "\":", "\",").Replace("\"","").Replace(" ","");
if (follower_list.Count < nOOfUserToScrape && !follower_list.Contains(user_name) && !ClGlobul.lstfollowerList_final.Contains(user_name))
{
follower_list.Add(user_name);
follower_list = follower_list.Distinct().ToList();
ClGlobul.lstfollowerList_final.Add(user_name);
try
{
GlobusLogHelper.log.Info("Scraped===>" + user_name);
string CSVData = "WithoutAccount".Replace(",", string.Empty) + "," + itemusername.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty);
GlobusFileHelper.ExportDataCSVFile(CSVHeader_Follower, CSVData, CSVPath_Follower);
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
try
{
DataBaseHandler.InsertQuery("insert into ScrapedUsername(Username) values('" + user_name + "')", "ScrapedUsername");
count++;
try
{
DataBaseHandler.InsertQuery("insert into tb_UserDetails(UserName,FriendsId) values('" + "WithoutAccount" + "','" + user_name + "')", "tb_UserDetails");
}
catch (Exception ex)
{
}
}
catch (Exception ex)
{
}
}
else
{
break;
}
}
}
if (Mindelay_ScarpeFollower != 0)
{
mindelay = Mindelay_ScarpeFollower;
}
if (Maxdelay_ScarpeFollower != 0)
{
maxdelay = Maxdelay_ScarpeFollower;
}
int delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay);
GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds ");
Thread.Sleep(delay * 1000);
if (GramFeed_UserFollower.Contains("next_url"))
{
value = true;
while (value)
{
if (GramFeed_UserFollower.Contains("next_url") && count < nOOfUserToScrape)
{
string hit_data = Utils.getBetween(GramFeed_UserFollower, "next_url\":", "\",").Replace("\u0026", "&").Replace("\\", "").Replace("u0026", "&").Replace("\"","").Replace(" ","");
GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(hit_data), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
if (string.IsNullOrEmpty(GramFeed_UserFollower))
{
if (lst_workingfinalproxy.Count > 0)
{
GramFeed_UserFollower = changeproxy(hit_data);
}
else
{
Thread.Sleep(1 * 3000);
GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl_withProxt(new Uri(Follower_Url), proxyAddress, int.Parse(proxyport), proxyusername, proxypasswrd);
}
}
string[] follower_regexlist2 = Regex.Split(GramFeed_UserFollower, "username");
foreach (string item in follower_regexlist2)
{
if (item.Contains("profile_picture"))
{
string user_name = Utils.getBetween(item, "\":", "\",").Replace("\"", "").Replace(" ", "");
if (follower_list.Count < nOOfUserToScrape && !follower_list.Contains(user_name) && !ClGlobul.lstfollowerList_final.Contains(user_name))
{
follower_list.Add(user_name);
follower_list = follower_list.Distinct().ToList();
ClGlobul.lstfollowerList_final.Add(user_name);
try
{
GlobusLogHelper.log.Info("Scraped===>" + user_name);
string CSVData = "WithoutAccount".Replace(",", string.Empty) + "," + itemusername.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty);
GlobusFileHelper.ExportDataCSVFile(CSVHeader_Follower, CSVData, CSVPath_Follower);
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
}
try
{
DataBaseHandler.InsertQuery("insert into ScrapedUsername(Username) values('" + user_name + "')", "ScrapedUsername");
count++;
try
{
DataBaseHandler.InsertQuery("insert into tb_UserDetails(UserName,FriendsId) values('" + "WithoutAccount" + "','" + user_name + "')", "tb_UserDetails");
}
catch (Exception ex)
{
}
}
catch (Exception ex)
{
}
}
else
{
break;
}
}
}
}
else
{
value = false;
}
if (minDelayScrapeUser != 0)
{
mindelay = minDelayScrapeUser;
}
if (maxDelayScrapeUser != 0)
{
maxdelay = maxDelayScrapeUser;
}
delay = RandomNumberGenerator.GenerateRandom(mindelay, maxdelay);
GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Delay For " + delay + " Seconds For ");
Thread.Sleep(delay * 1000);
}
}
}
//foreach (string itemusername in ClGlobul.listOfScrapeFollowerUserame)
//{
// string gramfeed_user_Url = "http://www.gramfeed.com/" + itemusername;
// Username_ScrapFollower = itemusername;
// List<string> follower_list = new List<string>();
// int count = 0;
// ClGlobul.switchAccount = false;
// string GramFeed_Userhit = obj_GlobusHttpHelper.getHtmlfromUrl(new Uri(gramfeed_user_Url), "");
// string client_Id = Utils.getBetween(GramFeed_Userhit, "client_id=", "&");
// string Id_Url = "https://api.instagram.com/v1/users/search?q=" + itemusername + "&client_id=" + client_Id;
// string Gram_Hit_second = obj_GlobusHttpHelper.getHtmlfromUrl(new Uri(Id_Url), "");
// string GramFeed_UserId = Utils.getBetween(Gram_Hit_second, "id\":\"", "\"");
// string Follower_Url = "https://api.instagram.com/v1/users/" + GramFeed_UserId + "/followed-by?client_id=" + client_Id;
// string GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl(new Uri(Follower_Url), "");
// string[] follower_regexlist = Regex.Split(GramFeed_UserFollower, "username");
// foreach (string item in follower_regexlist)
// {
// if (item.Contains("profile_picture"))
// {
// string user_name = Utils.getBetween(item, "\":\"", "\"");
// if (follower_list.Count < nOOfUserToScrape && !follower_list.Contains(user_name))
// {
// follower_list.Add(user_name);
// follower_list = follower_list.Distinct().ToList();
// ClGlobul.lstfollowerList_final.Add(user_name);
// // GlobusLogHelper.log.Info("Scraped===>" + user_name);
// try
// {
// GlobusLogHelper.log.Info("Scraped===>" + user_name);
// string CSVData = itemusername.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty);
// GlobusFileHelper.ExportDataCSVFile(CSVHeader_Followerwithoutlogin, CSVData, CSVPath_Followerwithoutlogin);
// }
// catch (Exception ex)
// {
// GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
// }
// }
// else
// {
// break;
// }
// }
// }
// if (GramFeed_UserFollower.Contains("next_url"))
// {
// value = true;
// while (value)
// {
// if (GramFeed_UserFollower.Contains("next_url") )
// {
// string hit_data = Utils.getBetween(GramFeed_UserFollower, "next_url\":\"", "\",").Replace("\u0026", "&").Replace("\\", "").Replace("u0026", "&");
// GramFeed_UserFollower = obj_GlobusHttpHelper.getHtmlfromUrl(new Uri(hit_data), "");
// string[] follower_regexlist2 = Regex.Split(GramFeed_UserFollower, "username");
// foreach (string item in follower_regexlist2)
// {
// if (item.Contains("profile_picture"))
// {
// string user_name = Utils.getBetween(item, "\":\"", "\"");
// if (follower_list.Count < nOOfUserToScrape && !follower_list.Contains(user_name))
// {
// follower_list.Add(user_name);
// follower_list = follower_list.Distinct().ToList();
// ClGlobul.lstfollowerList_final.Add(user_name);
// try
// {
// GlobusLogHelper.log.Info("Scraped===>" + user_name);
// string CSVData = itemusername.Replace(",", string.Empty) + "," + user_name.Replace(",", string.Empty);
// GlobusFileHelper.ExportDataCSVFile(CSVHeader_Followerwithoutlogin, CSVData, CSVPath_Followerwithoutlogin);
// }
// catch (Exception ex)
// {
// GlobusLogHelper.log.Error("Error:" + ex.StackTrace);
// }
// }
// else
// {
// break;
// }
// }
// }
// }
// else
// {
// value = false;
// }
// }
// }
//}
}
catch (Exception ex)
{
}
GlobusLogHelper.log.Info(" Process Completed By ");
}
catch(Exception ex)
{
}
}
