public void chekLicense() { #region FreeTrial try { ChilkatHttpHelpr objCjilkatHttpHelper = new ChilkatHttpHelpr(); string strdateTime_DataBase = ("2016-02-06 23:59:59").ToString();// string strdateTime_DataBase = ("2016-02-04 23:59:59").ToString(); DateTime dt = DateTime.Parse(strdateTime_DataBase); strdateTime_DataBase = dt.ToString("yyyy-MM-dd hh:mm:ss"); string dateTime = objCjilkatHttpHelper.GetHtml("http://licensing.facedominator.com/licensing/FD/Datetime.php"); DateTime dt_now = DateTime.Parse(dateTime); TimeSpan dt_Difference = dt_now.Subtract(dt); if (dt_Difference.Days >= 1) { //ModernDialog.ShowMessage("Your Trial Version of Software Has Been Expired!!","Warning Message",MessageBoxButton.OK,this.OwnedWindows); MessageBox.Show("Your Trial Version of Software Has Been Expired!!"); mainFormReference.Close(); return; } } catch { } #endregion }
/// <summary> /// This Method is used for check given proxy has working or not And Return those Proxy set for use in their method /// </summary> /// <param name="IPaddress">Given IpAddress </param> /// <param name="Port">Given Port</param> /// <param name="Username">Given Username</param> /// <param name="Password">Given Password</param> /// <param name="CheckingUrl">Given Url where we check the Proxy</param> /// <returns></returns> public Proxies.Proxy GetSingleUserProxy(string IPaddress, string Port, string Username, string Password, string CheckingUrl) { ChilkatHttpHelpr objChilkatHelper = new ChilkatHttpHelpr(); Proxy objProxy = new Proxy(); try { System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch(); string respnceTime = string.Empty; timer.Start(); string Responce = objChilkatHelper.GetHtmlProxy(CheckingUrl, IPaddress, Port, Username, Password); timer.Stop(); TimeSpan timeTaken = timer.Elapsed; respnceTime = timeTaken.ToString(); if (!string.IsNullOrEmpty(Responce) && objChilkatHelper.http.LastStatus != 404 && objChilkatHelper.http.LastStatus != 500 && !Responce.Contains("Internet Explorer cannot display the webpage")) { objProxy.IpAddress = IPaddress; objProxy.Port = Port; objProxy.UserName = Username; objProxy.Password = Password; } else { } } catch (Exception ex) { GlobusLogHelper.log.Error("Error : " + ex.StackTrace); } return(objProxy); }
/// <summary> /// This Method is used for check given proxy has working or not ? /// </summary> /// <param name="IPaddress">Given IpAddress </param> /// <param name="Port">Given Port</param> /// <param name="Username">Given Username</param> /// <param name="Password">Given Password</param> /// <param name="CheckingUrl">Given Url where we check the Proxy</param> /// <returns></returns> public bool CheckProxy(string IPaddress, string Port, string Username, string Password, string CheckingUrl) { ChilkatHttpHelpr objChilkatHelper = new ChilkatHttpHelpr(); bool status = false; try { System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch(); string ResponseTime = string.Empty; timer.Start(); string Response = objChilkatHelper.GetHtmlProxy(CheckingUrl, IPaddress, Port, Username, Password); timer.Stop(); TimeSpan timeTaken = timer.Elapsed; ResponseTime = timeTaken.ToString(); if (!string.IsNullOrEmpty(Response) && objChilkatHelper.http.LastStatus != 404 && objChilkatHelper.http.LastStatus != 500 && !Response.Contains("Internet Explorer cannot display the webpage")) { status = true; } else { status = false; } } catch (Exception ex) { GlobusLogHelper.log.Error("Error : " + ex.StackTrace); } return(status); }
public string RegisterUser(string Username, string Password, string cpuID, string TransactionID, string Email, string servr) { ChilkatHttpHelpr HttpHelpr = new ChilkatHttpHelpr(); string res = string.Empty; try { string regUrl = "http://" + servr + "/register.php?user="******"&pass="******"&cpid=" + cpuID + "&transid=" + TransactionID + "&email=" + Email + "&LicType=" + Globals.licType + " "; res = HttpHelpr.GetHtml("http://" + servr + "/register.php?user="******"&pass="******"&cpid=" + cpuID + "&transid=" + TransactionID + "&email=" + Email + "&LicType=" + Globals.licType + ""); if (string.IsNullOrEmpty(res)) { System.Threading.Thread.Sleep(1000); res = HttpHelpr.GetHtml("http://" + servr + "/register.php?user="******"&pass="******"&cpid=" + cpuID + "&transid=" + TransactionID + "&email=" + Email + "&LicType=" + Globals.licType + " "); } if (string.IsNullOrEmpty(res)) { MessageBox.Show("Error Connecting to Facedominator Server,Please check if www.facedominator.com is opening for you."); // Application.Exit(); } } catch (Exception ex) { MessageBox.Show(ex.Message); } return(res); }
public static string GetUserIDFromUsername(string username, out string Status) { string GetStatus = string.Empty; Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper(); clsDBQueryManager DB = new clsDBQueryManager(); DataSet ds = DB.GetUserId(username); string user_id = string.Empty; if (ds.Tables[0].Rows.Count > 0) { foreach (DataRow dataRow in ds.Tables["tb_UsernameDetails"].Rows) { user_id = dataRow.ItemArray[0].ToString(); Status = "No Error"; return(user_id); } } try { string id = string.Empty; ChilkatHttpHelpr httpHelper1 = new ChilkatHttpHelpr(); string pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username); if (!pagesource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.") && !pagesource.Contains("Sorry, that page does not exist") && !pagesource.Contains("User has been suspended")) { int length = pagesource.IndexOf("</id>"); id = pagesource.Substring(pagesource.IndexOf("<id>"), length - pagesource.IndexOf("<id>")).Replace("<id>", ""); user_id = id; GetStatus = "No Error"; } else if (pagesource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.")) { GetStatus = "Rate limit exceeded"; } else if (pagesource.Contains("Sorry, that page does not exist")) { GetStatus = "Sorry, that page does not exist"; } else if (pagesource.Contains("User has been suspended")) { GetStatus = "User has been suspended"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserIDFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserIDFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); GetStatus = "Error"; } Status = GetStatus; return(user_id); }
public List <string> GetFollowers(string userID, out string ReturnStatus) { try { string FollowingUrl = string.Empty; if (NumberHelper.ValidateNumber(userID)) { FollowingUrl = "https://api.twitter.com/1/followers/ids.json?cursor=-1&id=" + userID + "";//"https://api.twitter.com/1/friends/ids.json?cursor=-1&screen_name=SocioPro"; } else { FollowingUrl = "https://api.twitter.com/1/followers/ids.json?cursor=-1&screen_name=" + userID + "";//"https://api.twitter.com/1/friends/ids.json?cursor=-1&screen_name=SocioPro"; } #region gs http helper code //https://api.twitter.com/1/following/ids.json?cursor=-1&screen_name=SocioPro //string Data = globushttpHelper.getHtmlfromUrl(new Uri(FollowingUrl), "", ""); #endregion ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); string Data = HttpHelper.GetHtml(FollowingUrl); if (!Data.Contains("Rate limit exceeded") && !Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}")) { int FirstPoint = Data.IndexOf("["); int SecondPoint = Data.IndexOf("]"); string FollowingIds = Data.Substring(FirstPoint, SecondPoint - FirstPoint).Replace("[", string.Empty).Replace("]", string.Empty); List <string> lstIds = FollowingIds.Split(',').ToList(); ReturnStatus = "No Error"; return(lstIds); } else if (Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}")) { ReturnStatus = "Sorry, that page does not exist :" + userID; return(new List <string>()); } else { ReturnStatus = "Error"; return(new List <string>()); } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowers() -- " + userID + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowers() -- " + userID + " --> " + ex.Message, Globals.Path_TwtErrorLogs); ReturnStatus = "Error"; return(new List <string>()); } }
/// <summary> Get User details for While/blacklist User /// Get User details for While/blacklist User /// </summary> /// <param name="username">User Screan Name OR User ID</param> /// <returns>Details of Users</returns> #region Get User details for While/blacklist User public static Dictionary <string, string> GetUserDetails(string username) { ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr(); Dictionary <string, string> dataLst = new Dictionary <string, string>(); try { string PageSource = null; if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username)) { PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?user_id=" + username + "&include_entities=true"); } else { PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?screen_name=" + username + "&include_entities=true"); } JObject jobj = JObject.Parse(PageSource); string id = jobj["id"].ToString().Replace("\"", string.Empty); dataLst.Add("id", id); string name = jobj["name"].ToString().Replace("\"", string.Empty); dataLst.Add("name", name); string screen_name = jobj["screen_name"].ToString().Replace("\"", string.Empty); dataLst.Add("screen_name", screen_name); string location = jobj["location"].ToString().Replace("\"", string.Empty); dataLst.Add("location", location); string description = jobj["description"].ToString().Replace("\"", string.Empty); dataLst.Add("description", description); string followers_count = jobj["followers_count"].ToString().Replace("\"", string.Empty); dataLst.Add("followers_count", followers_count); string friends_count = jobj["friends_count"].ToString().Replace("\"", string.Empty); dataLst.Add("friends_count", friends_count); string statuses_count = jobj["statuses_count"].ToString().Replace("\"", string.Empty); dataLst.Add("statuses_count", statuses_count); string created_at = jobj["created_at"].ToString().Replace("\"", string.Empty); dataLst.Add("created_at", created_at); string time_zone = jobj["time_zone"].ToString().Replace("\"", string.Empty); dataLst.Add("time_zone", time_zone); string profile_image_url = jobj["profile_image_url"].ToString().Replace("\"", string.Empty); dataLst.Add("profile_image_url", profile_image_url); } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return(dataLst); }
public List <string> GetHashTags(out string returnStatus) { List <string> HashTags = new List <string>(); try { //string pagesource = globushttpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/trends/daily.json"), "", ""); ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); string pagesource = HttpHelper.GetHtml("https://api.twitter.com/1/trends/daily.json"); if (!pagesource.Contains("Rate limit exceeded")) { string[] array = Regex.Split(pagesource, "\"name\":"); array = array.Skip(1).ToArray(); foreach (string item in array) { try { int startindex = item.IndexOf("\""); string Start = item.Substring(startindex); int endIndex = Start.IndexOf("\","); string End = Start.Substring(0, endIndex).Replace("\"", ""); string HashTag = End; if (HashTag.Contains("#")) { HashTags.Add(HashTag); } } catch (Exception ex) { returnStatus = "Error"; return(new List <string>()); } } returnStatus = "No Error"; return(HashTags); } else { returnStatus = "Error"; return(new List <string>()); } } catch (Exception ex) { returnStatus = "Error"; return(new List <string>()); } }
public string GetUserLastTweetDate(string userid) { string date = ""; try { string user_id = string.Empty; string searchURL = string.Empty; if (!NumberHelper.ValidateNumber(userid)) { searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + user_id + "&count=" + TweetExtractCount; } else if (NumberHelper.ValidateNumber(userid)) { searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&screen_name=" + user_id + "&count=" + TweetExtractCount; } //string searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + user_id + "&count=" + TweetExtractCount; ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr(); //string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); string res_Get_searchURL = httpHelper.GetHtml(searchURL); string[] splitRes = Regex.Split(res_Get_searchURL, "{\"created_at");//Regex.Split(res_Get_searchURL, "{\"created_at\""); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { //string text = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "<text>"); string modified_Item = "{\"created_at" + item; date = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "created_at");//Globussoft.GlobusHttpHelper.parseText(item); if (date.Contains("+")) { date = date.Remove(date.IndexOf("+")).Trim(); } if (!string.IsNullOrEmpty(date)) { return(date); } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserLastTweetDate() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserLastTweetDate() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return(date); }
public static string GetUserNameFromUserId(string userid) { string username = string.Empty; ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr(); try { string PageSource = string.Empty; if (!string.IsNullOrEmpty(userid) && NumberHelper.ValidateNumber(userid)) { PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + userid + "&include_entities=true"); if (!PageSource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.")) { try { int startIndex = PageSource.IndexOf("<screen_name>"); if (startIndex > 0) { string Start = PageSource.Substring(startIndex); int endIndex = Start.IndexOf("</screen_name>"); string End = Start.Substring(0, endIndex); username = End.Replace("<screen_name>", ""); } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } } else { username = "******"; } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return(username); }
public string GetUserLastTweetDate(string userid) { string date = ""; try { string user_id = string.Empty; string searchURL = string.Empty; if (!NumberHelper.ValidateNumber(userid)) { searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + user_id + "&count=" + TweetExtractCount; } else if (NumberHelper.ValidateNumber(userid)) { searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&screen_name=" + user_id + "&count=" + TweetExtractCount; } //string searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + user_id + "&count=" + TweetExtractCount; ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr(); //string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); string res_Get_searchURL = httpHelper.GetHtml(searchURL); string[] splitRes = Regex.Split(res_Get_searchURL, "{\"created_at");//Regex.Split(res_Get_searchURL, "{\"created_at\""); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { //string text = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "<text>"); string modified_Item = "{\"created_at" + item; date = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "created_at");//Globussoft.GlobusHttpHelper.parseText(item); if (date.Contains("+")) { date = date.Remove(date.IndexOf("+")).Trim(); } if (!string.IsNullOrEmpty(date)) { return date; } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserLastTweetDate() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserLastTweetDate() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return date; }
public string Login() { Log("[ " + DateTime.Now + " ] => [ Logging in with Account : " + Username + " ]"); string Status = "Failed"; try { string firstUrl = "https://api.instagram.com/oauth/authorize/?client_id=9d836570317f4c18bca0db6d2ac38e29&redirect_uri=http://websta.me/&response_type=code&scope=comments+relationships+likes"; //https://instagram.com/oauth/authorize/?client_id=9d836570317f4c18bca0db6d2ac38e29&redirect_uri=http://websta.me/&response_type=code&scope=comments+relationships+likes string secondURL = "https://instagram.com/oauth/authorize/?client_id=9d836570317f4c18bca0db6d2ac38e29&redirect_uri=http://websta.me/&response_type=code&scope=comments+relationships+likes"; string res_secondURL = string.Empty; ChilkatHttpHelpr objchilkat = new ChilkatHttpHelpr(); if (!string.IsNullOrEmpty(proxyAddress) && !string.IsNullOrEmpty(proxyPort)) { try { //res_secondURL = httpHelper.getHtmlfromUrlProxy(new Uri(secondURL), proxyAddress, int.Parse(proxyPort), proxyUsername, proxyPassword); res_secondURL = objchilkat.GetHtmlProxy(secondURL, proxyAddress, proxyPort, proxyUsername, proxyPassword); } catch { }; } else { res_secondURL = httpHelper.getHtmlfromUrl(new Uri(secondURL), ""); } //string authlogin = "******"; //string res_nextUrl1 = httpHelper.getHtmlfromUrl(new Uri(authlogin), ""); string nextUrl = string.Empty; string res_nextUrl = string.Empty; if (!string.IsNullOrEmpty(res_secondURL)) { nextUrl = "https://instagram.com/accounts/login/?force_classic_login=&next=/oauth/authorize/%3Fclient_id%3D9d836570317f4c18bca0db6d2ac38e29%26redirect_uri%3Dhttp%3A//websta.me/%26response_type%3Dcode%26scope%3Dcomments%2Brelationships%2Blikes"; res_nextUrl = httpHelper.getHtmlfromUrl(new Uri(nextUrl), "");//postFormDataProxy } else { Log("[ " + DateTime.Now + " ] => [ Logged in Failed with Account :" + Username + " ]"); Status = "Failed"; this.LoggedIn = false; } //string nextUrl = "https://instagram.com/accounts/login/?force_classic_login=&next=/oauth/authorize/%3Fclient_id%3D9d836570317f4c18bca0db6d2ac38e29%26redirect_uri%3Dhttp%3A//websta.me/%26response_type%3Dcode%26scope%3Dcomments%2Brelationships%2Blikes"; //string res_nextUrl = httpHelper.getHtmlfromUrl(new Uri(nextUrl), "");//postFormDataProxy //int FirstPointToken_nextUrl1 = res_nextUrl.IndexOf("authorize"); //string FirstTokenSubString_nextUrl1 = res_nextUrl.Substring(FirstPointToken_nextUrl1); //int SecondPointToken_nextUrl1 = FirstTokenSubString_nextUrl1.IndexOf("%"); //this.Token = FirstTokenSubString_nextUrl1.Substring(0, SecondPointToken_nextUrl1).Replace("Authorize", string.Empty).Replace("value=", string.Empty).Replace("\"", string.Empty).Replace("'", string.Empty).Trim(); try { int FirstPointToken_nextUrl = res_nextUrl.IndexOf("csrfmiddlewaretoken");//csrfmiddlewaretoken string FirstTokenSubString_nextUrl = res_nextUrl.Substring(FirstPointToken_nextUrl); int SecondPointToken_nextUrl = FirstTokenSubString_nextUrl.IndexOf("/>"); this.Token = FirstTokenSubString_nextUrl.Substring(0, SecondPointToken_nextUrl).Replace("csrfmiddlewaretoken", string.Empty).Replace("value=", string.Empty).Replace("\"", string.Empty).Replace("'", string.Empty).Trim(); } catch { }; string login = "******"; //string authlogin= "******"; //string postauthorize = "csrfmiddlewaretoken=" + this.Token + "&allow=Authorize"; //http://websta.me/?code=90786edc0a4844b79a3d9289d27360d5 string postdata_Login = "******" + this.Token + "&username="******"&password="******""; string res_postdata_Login = httpHelper.postFormData(new Uri(login), postdata_Login, login, ""); string autho = "https://instagram.com/oauth/authorize/?scope=comments+likes+relationships&redirect_uri=http%3A%2F%2Fwww.gramfeed.com%2Foauth%2Fcallback%3Fpage%3D&response_type=code&client_id=b59fbe4563944b6c88cced13495c0f49"; if (res_postdata_Login.Contains("Please enter a correct username and password")) { Status = "Failed"; this.LoggedIn = false; } else if (res_postdata_Login.Contains("requesting access to your Instagram account") || postdata_Login.Contains("is requesting to do the following")) { Status = "AccessIssue"; } else if (res_postdata_Login.Contains("logout") || postdata_Login.Contains("LOG OUT")) { Log("[ " + DateTime.Now + " ] => [ Logged in with Account :" + Username + " ]"); Status = "Success"; this.LoggedIn = true; } //nameval.Clear(); return(Status); } catch (Exception ex) { return(ex.Message); } }
private void btnStart_Searching_Click(object sender, EventArgs e) { AllOfTheseWords = (txtAllofTheseKeywords.Text).ToString(); ThisExtractPhrase = (txtThisExactPhrase.Text).ToString(); AnyOfTheseWords = (txtAnyOfTheseWords.Text).ToString(); TheseHashTags = (txtTheseHashTags.Text).ToString(); NoneOfTheseWords = (txtNoneofTheseWords.Text).ToString(); FromTheseAccounts = (txtFromTheseAccounts.Text).ToString(); ToTheseAccounts = (txtToTheseAccounts.Text).ToString(); MentionTheseAccounts = (txtMentioningTheseAccounts.Text).ToString(); NearThisPlace = (txtNearThisPlace.Text).ToString(); AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => Process Started"); try { if (string.IsNullOrEmpty(ThisExtractPhrase)) { ThisExtractPhrase = ""; } else { ThisExtractPhrase = "%20%22" + ThisExtractPhrase; } } catch { } try { if (string.IsNullOrEmpty(AnyOfTheseWords)) { AnyOfTheseWords = ""; } else { AnyOfTheseWords = "%22%20" + AnyOfTheseWords; } } catch { } try { if (string.IsNullOrEmpty(TheseHashTags)) { TheseHashTags = ""; } else { TheseHashTags = "%20%23" + TheseHashTags; } } catch { } try { if (string.IsNullOrEmpty(NoneOfTheseWords)) { NoneOfTheseWords = ""; } else { NoneOfTheseWords = "%20-" + NoneOfTheseWords; } } catch { } try { if (string.IsNullOrEmpty(FromTheseAccounts)) { FromTheseAccounts = ""; } else { FromTheseAccounts = "%20from%3A" + FromTheseAccounts; } } catch { } try { if (string.IsNullOrEmpty(ToTheseAccounts)) { ToTheseAccounts = ""; } else { ToTheseAccounts = "%20to%3A" + ToTheseAccounts; } } catch { } try { if (string.IsNullOrEmpty(MentionTheseAccounts)) { MentionTheseAccounts = ""; } else { MentionTheseAccounts = "%20%40" + MentionTheseAccounts; } } catch { } try { if (string.IsNullOrEmpty(NearThisPlace)) { NearThisPlace = ""; } else { NearThisPlace = "%20near%3A%22" + NearThisPlace; } } catch { } try { if (!string.IsNullOrEmpty(txtAllofTheseKeywords.Text)) { #region Commented //try //{ // string Url = "https://twitter.com/search?f=realtime&q=" + AllOfTheseWords + ThisExtractPhrase + AnyOfTheseWords + NoneOfTheseWords + TheseHashTags + _selectedLanguage + FromTheseAccounts + ToTheseAccounts + MentionTheseAccounts + NearThisPlace + "%22%20within%3A15mi&src=typd"; // string response = _GlobusHttpHelper.getHtmlfromUrl(new Uri(Url), "", ""); //} //catch { } public List<StructTweetIDs> NewKeywordStructDataForSearchByKeyword(string keyword) #endregion { try { BaseLib.GlobusRegex regx = new GlobusRegex(); int counter = 0; string res_Get_searchURL = string.Empty; string searchURL = string.Empty; string maxid = string.Empty; string TweetId = string.Empty; string text = string.Empty; string ProfileName = string.Empty; string Location = string.Empty; string Bio = string.Empty; string website = string.Empty; string NoOfTweets = string.Empty; string Followers = string.Empty; string Followings = string.Empty; int noOfRecords = 0; try { noOfRecords = int.Parse(txtNoOfRecords.Text); } catch { } startAgain: if (counter == 0) { searchURL = "https://twitter.com/i/search/timeline?q=" + AllOfTheseWords + ThisExtractPhrase + AnyOfTheseWords + NoneOfTheseWords + TheseHashTags + _selectedLanguage + FromTheseAccounts + ToTheseAccounts + MentionTheseAccounts + NearThisPlace + "%22%20within%3A15mi&src=typd" + "&f=realtime"; counter++; } else { searchURL = "https://twitter.com/i/search/timeline?q=" + AllOfTheseWords + ThisExtractPhrase + AnyOfTheseWords + NoneOfTheseWords + TheseHashTags + _selectedLanguage + FromTheseAccounts + ToTheseAccounts + MentionTheseAccounts + NearThisPlace + "%22%20within%3A15mi&src=typd" + "&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; } try { res_Get_searchURL = _GlobusHttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => Finding results for entered details "); if (string.IsNullOrEmpty(res_Get_searchURL)) { res_Get_searchURL = _GlobusHttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } try { //string sjss = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor"); //splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("refresh_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } if (item.Contains("scroll_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } } } catch (Exception) { } } catch (Exception ex) { System.Threading.Thread.Sleep(2000); res_Get_searchURL = _GlobusHttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } // && !res_Get_searchURL.Contains("has_more_items\":false") if (!string.IsNullOrEmpty(res_Get_searchURL)) { //string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); //Regex.Split(res_Get_searchURL, "\"in_reply_to_status_id_str\""); string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("data-screen-name=") && !item.Contains("js-actionable-user js-profile-popup-actionable")) { //var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL); //string DataHtml = (string)avc["items_html"]; } else { continue; } string modified_Item = "\"from_user\"" + item; string id = ""; try { int startIndex = item.IndexOf("data-user-id="); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); id = end; //lst_structTweetIDs.Add(id); AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => User Id " + id); } catch (Exception ex) { id = "null"; //Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); } string from_user_id = ""; try { int startIndex = item.IndexOf("data-screen-name=\\\""); string start = item.Substring(startIndex).Replace("data-screen-name=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); from_user_id = end; AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => User ScreenName " + from_user_id); } catch (Exception ex) { from_user_id = "null"; // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); } string tweetUserid = string.Empty; try { int startIndex = item.IndexOf("=\\\""); string start = item.Substring(startIndex).Replace("=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end; AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => Tweet Id " + tweetUserid); } catch (Exception ex) { from_user_id = "null"; } ///Tweet Text #region Commented //try //{ // int startindex = item.IndexOf("js-tweet-text tweet-text\""); // if (startindex == -1) // { // startindex = 0; // startindex = item.IndexOf("js-tweet-text tweet-text"); // } // string start = item.Substring(startindex).Replace("js-tweet-text tweet-text\"", "").Replace("js-tweet-text tweet-text tweet-text-rtl\"", ""); // int endindex = start.IndexOf("</p>"); // if (endindex == -1) // { // endindex = 0; // endindex = start.IndexOf("stream-item-footer"); // } // string end = start.Substring(0, endindex); // end = regx.StripTagsRegex(end); // text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); // text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); // string[] array = Regex.Split(text, "http"); // text = string.Empty; // foreach (string itemData in array) // { // if (!itemData.Contains("t.co")) // { // string data = string.Empty; // if (itemData.Contains("//")) // { // data = ("http" + itemData).Replace(" span ", string.Empty); // if (!text.Contains(itemData.Replace(" ", "")))// && !data.Contains("class") && !text.Contains(data)) // { // text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty); // } // } // else // { // if (!text.Contains(itemData.Replace(" ", ""))) // { // text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("js-tweet-text tweet-text", ""); // } // } // } // } //} //catch { }; #endregion twtboardpro.TwitterDataScrapper.StructTweetIDs structTweetIDs = new twtboardpro.TwitterDataScrapper.StructTweetIDs(); if (id != "null") { structTweetIDs.ID_Tweet = tweetUserid; structTweetIDs.ID_Tweet_User = id; structTweetIDs.username__Tweet_User = from_user_id; structTweetIDs.wholeTweetMessage = text; lst_structTweetIDs.Add(structTweetIDs); } //if (!File.Exists(Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv")) //{ // GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS", Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"); //} { ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr(); GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + from_user_id), "", ""); string Responce = ProfilePageSource; #region Convert HTML to XML string xHtml = objChilkat.ConvertHtmlToXml(Responce); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion int counterdata = 0; xBeginSearchAfter = null; string dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterdata == 0) { ProfileName = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else if (counterdata == 1) { website = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else { break; } xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor"); } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim(); break; } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Location = xNode.AccumulateTagContent("text", "script|style"); break; } int counterData = 0; xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterData == 0) { // NoOfTweets = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileNav-value"); NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty); counterData++; } else if (counterData == 1) { Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty); counterData++; } else if (counterData == 2) { Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty); counterData++; } else { break; } //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor"); } if (!string.IsNullOrEmpty(from_user_id) && tweetUserid != "null") { string Id_user = tweetUserid.Replace("}]", string.Empty).Trim(); Globals.lstScrapedUserIDs.Add(Id_user); // GlobusFileHelper.AppendStringToTextfileNewLine(id + "," + from_user_id + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"); // Log("[ " + DateTime.Now + " ] => [ " + from_user_id + "," + Id_user + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]"); } } lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList(); if (lst_structTweetIDs.Count >= noOfRecords) { // return lst_structTweetIDs; } } if (lst_structTweetIDs.Count <= noOfRecords) { maxid = lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet; if (res_Get_searchURL.Contains("has_moreitems\":false")) { } else { goto startAgain; } } else { if (res_Get_searchURL.Contains("has_more_items\":false")) { } else goto startAgain; } } } catch (Exception ex) { } } } } catch { } }
public static string GetPhotoFromUsername(string username) { //Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper(); ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr(); string user_id = string.Empty; string containsImage = "false"; try { string ImageLink = string.Empty; string PageSource = string.Empty; //http://a0.twimg.com/sticky/default_profile_images/default_profile_0_normal.png if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username)) { //PageSource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?user_id=" + username + "&include_entities=true"), "", ""); PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + username + "&include_entities=true"); } else { //PageSource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?screen_name=" + username + "&include_entities=true"), "", ""); PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username + "&include_entities=true"); } if (!string.IsNullOrEmpty(PageSource)) { try { int indexStart = PageSource.IndexOf("<profile_image_url>"); if (PageSource.Contains("<profile_image_url>")) { string start = PageSource.Substring(indexStart); int endIndex = start.IndexOf("</profile_image_url>"); string end = start.Substring(0, endIndex); ImageLink = end.Replace("<profile_image_url>", ""); } if (!string.IsNullOrEmpty(ImageLink) && ImageLink.Contains("/sticky/default_profile_images/default_profile")) { containsImage = "false"; } else if (PageSource.Contains("Sorry, that page does not exist")) { containsImage = "false"; } else { containsImage = "true"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } } else if (PageSource.Contains("Clients may not make more than 150 requests per hour") && PageSource.Contains("Rate limit exceeded")) { containsImage = "Rate limit exceeded"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return(containsImage); }
public void getpageSourceFromProxy(object item) { if (proxyStop) { return; } try { Thread.CurrentThread.IsBackground = true; lstProxyThread.Add(Thread.CurrentThread); lstProxyThread = lstProxyThread.Distinct().ToList(); } catch { } countParseProxiesThreads++; Array Item_value = (Array)item; string ClGlobul_ProxyList_item = (string)Item_value.GetValue(0); DemoStagramPro.frm_stagram frm1 = new DemoStagramPro.frm_stagram(); Globussoft.GlobDramProHttpHelper GlobusHttpHelper = new Globussoft.GlobDramProHttpHelper(); ChilkatHttpHelpr objchilkat = new ChilkatHttpHelpr(); string proxyad = string.Empty; string proxyport = string.Empty; string proxyusername = string.Empty; string proxyPassword = string.Empty; string pagesource1 = string.Empty; string pagesource = string.Empty; try { string[] proxyLst = ClGlobul_ProxyList_item.Split(':'); if (proxyLst.Count() > 3) { proxyad = proxyLst[0]; proxyport = proxyLst[1]; proxyusername = proxyLst[2]; proxyPassword = proxyLst[3]; } else if (proxyLst.Count() > 0 && proxyLst.Count() < 3) { proxyad = proxyLst[0]; proxyport = proxyLst[1]; } else { return; } try { if (proxyStop) { return; } //pagesource1 = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri("http://websta.me/login"), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword); pagesource1 = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri("http://web.stagram.com/"), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword); } catch { }; try { // pagesource1 = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri("http://web.stagram.com/"), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword); pagesource1 = objchilkat.GetHtmlProxy("http://web.stagram.com/", proxyad, proxyport, proxyusername, proxyPassword); } catch { }; //int FirstPointClientId = pagesource1.IndexOf("client_id="); //string FirstClientIdSubString = pagesource1.Substring(FirstPointClientId); //int SecondPointClientId = FirstClientIdSubString.IndexOf("&redirect_uri="); //string ClientId = FirstClientIdSubString.Substring(0, SecondPointClientId).Replace("'", string.Empty).Replace("client_id=", string.Empty).Trim(); //string LoginUrl = "https://instagram.com/accounts/login/?next=/oauth/authorize/%3Fclient_id%3D" + ClientId + "%26redirect_uri%3Dhttp%253A%252F%252Fweb.stagram.com%252F%26response_type%3Dcode%26scope%3Dlikes%2Bcomments%2Brelationships"; //pagesource = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri(LoginUrl), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword); //if (string.IsNullOrEmpty(pagesource)) //{ // pagesource = string.Empty; // pagesource = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri(LoginUrl), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword); //} //ADD in List list of Finally chacked..... if (!string.IsNullOrEmpty(pagesource1)) { if (proxyStop) { return; } addInFinalCheckedProxyist(proxyad, proxyport, proxyusername, proxyPassword, pagesource1); } else { if (proxyStop) { return; } DemoStagramPro.ClGlobul.isProxyCheckComplete = true; LoggerProxy("[ " + DateTime.Now + " ] => [ Proxy Is not Working : " + proxyad + ":" + proxyport + ":" + proxyusername + ":" + proxyPassword + " ]"); lock (lockerforNonWorkingProxies) { GramBoardProFileHelper.AppendStringToTextfileNewLine(proxyad + ":" + proxyport + ":" + proxyusername + ":" + proxyPassword, GramBoardProFileHelper.NonWorkingProxiesList); } } } catch (Exception) { if (proxyStop) { return; } LoggerProxy("[ " + DateTime.Now + " ] => [ Proxy Is not Working : " + proxyad + ":" + proxyport + ":" + proxyusername + ":" + proxyPassword + " ]"); lock (lockerforNonWorkingProxies) { GramBoardProFileHelper.AppendStringToTextfileNewLine(proxyad + ":" + proxyport + ":" + proxyusername + ":" + proxyPassword, GramBoardProFileHelper.NonWorkingProxiesList); } } finally { lock (lockerforProxies) { countParseProxiesThreads--; Monitor.Pulse(lockerforProxies); } Proxystatus--; if (Proxystatus == 0) { LoggerProxy("[ " + DateTime.Now + " ] => [ PROCESS COMPLETED ]"); LoggerProxy("-----------------------------------------------------------------------------------------------------------------------------------"); } } }
/// <summary> Get User details for While/blacklist User /// Get User details for While/blacklist User /// </summary> /// <param name="username">User Screan Name OR User ID</param> /// <returns>Details of Users</returns> /// #region Get User details for While/blacklist User //public static Dictionary<string, string> GetUserDetails(string username, out string status) //{ //// ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr(); // Dictionary<string, string> dataLst = new Dictionary<string, string>(); //// try //// { //// string PageSource = null; //// if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username)) //// { //// //PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?user_id=" + username + "&include_entities=true"); //// } //// else //// { //// //PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?screen_name=" + username + "&include_entities=true"); //// string url = string.Format("https://twitter.com/" + username); //// PageSource = httpHelper.GetHtml(url); //// } //// if (PageSource.Contains("error\":\"User has been suspended")) //// { //// status = "User has been suspended"; //// return dataLst; //// } //// else if (PageSource.Contains("message\":\"Sorry, that page does not exist")) //// { //// status = "Sorry, that page does not exist"; //// return dataLst; //// } //// else if (PageSource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.")) //// { //// status = "Rate limit exceeded. Clients may not make more than 150 requests per hour."; //// return dataLst; //// } //// if (!string.IsNullOrEmpty(PageSource)) //// { //// string id = string.Empty; //// string name = string.Empty; //// string screen_name = string.Empty; //// string location = string.Empty; //// string NoOfTweet = string.Empty; //// string followers_count = string.Empty; //// string friends_count = string.Empty; //// string profile_image_url = string.Empty; //// if (PageSource.Contains("data-user-id=")) //// { //// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "data-user-id="); //// if (getDataInArr.Count() > 0) //// { //// foreach (var item in getDataInArr) //// { //// int startindex = item.IndexOf("<strong>"); //// int endindex = item.IndexOf("</strong>"); ; //// NoOfTweet = item.Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty); //// } //// } //// } //// if (PageSource.Contains("tweet_stats")) //// { //// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "tweet_stats"); //// if (getDataInArr.Count() > 0) //// { //// int startindex = getDataInArr[1].IndexOf("<strong>"); //// int endindex = getDataInArr[1].IndexOf("</strong>"); ; //// NoOfTweet = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty); //// } //// } //// if (PageSource.Contains("following_stats")) //// { //// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "following_stats"); //// if (getDataInArr.Count() > 0) //// { //// int startindex = getDataInArr[1].IndexOf("<strong>"); //// int endindex = getDataInArr[1].IndexOf("</strong>"); ; //// friends_count = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty); //// } //// } //// if (PageSource.Contains("follower_stats")) //// { //// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "follower_stats"); //// if (getDataInArr.Count() > 0) //// { //// int startindex = getDataInArr[1].IndexOf("<strong>"); //// int endindex = getDataInArr[1].IndexOf("</strong>"); //// followers_count = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty); //// } //// } //// if (PageSource.Contains("data-url")) //// { //// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "data-url"); //// if (getDataInArr.Count() > 0) //// { //// int startindex = getDataInArr[1].IndexOf("=\""); //// int endindex = getDataInArr[1].IndexOf(">"); ; //// profile_image_url = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("target=\"_blank", string.Empty).Replace("\"", string.Empty).Replace("=", string.Empty).Trim(); //// } //// } //// //string id = jobj["id"].ToString().Replace("\"", string.Empty); //// dataLst.Add("id", id); //// //string name = jobj["name"].ToString().Replace("\"", string.Empty); //// dataLst.Add("name", name); //// //string screen_name = jobj["screen_name"].ToString().Replace("\"", string.Empty); //// dataLst.Add("screen_name", screen_name); //// //string location = jobj["location"].ToString().Replace("\"", string.Empty); //// //dataLst.Add("location", location); //// //string description = jobj["description"].ToString().Replace("\"", string.Empty); //// //dataLst.Add("description", description); //// //string followers_count = jobj["followers_count"].ToString().Replace("\"", string.Empty); //// //dataLst.Add("followers_count", followers_count); //// //string friends_count = jobj["friends_count"].ToString().Replace("\"", string.Empty); //// //dataLst.Add("friends_count", friends_count); //// //string statuses_count = jobj["statuses_count"].ToString().Replace("\"", string.Empty); //// //dataLst.Add("statuses_count", statuses_count); //// //string created_at = jobj["created_at"].ToString().Replace("\"", string.Empty); //// //dataLst.Add("created_at", created_at); //// //string time_zone = jobj["time_zone"].ToString().Replace("\"", string.Empty); //// //dataLst.Add("time_zone", time_zone); //// //string profile_image_url = jobj["profile_image_url"].ToString().Replace("\"", string.Empty); //// //dataLst.Add("profile_image_url", profile_image_url); //// } //// } //// catch (Exception ex) //// { //// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); //// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); //// } //// //Assign Status // status = "Ok"; // return dataLst; //} public static Dictionary<string, string> GetUserDetails(string username, out string status) { TwitterDataScrapper twiterDataScrapper = new TwitterDataScrapper(); Dictionary<string, string> dataLst = new Dictionary<string, string>(); string id = string.Empty; string name = string.Empty; string screen_name = string.Empty; string location = string.Empty; string description = string.Empty; string NoOfTweet = string.Empty; string followers_count = string.Empty; string following_count = string.Empty; string profile_image_url = string.Empty; //string ProfileName = string.Empty; //string Location = string.Empty; //string Bio = string.Empty; string website = string.Empty; //string NoOfTweets = string.Empty; //string Followers = string.Empty; //string Followings = string.Empty; //try //{ ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr(); GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + username), "", ""); string Responce = ProfilePageSource; #region Convert HTML to XML string xHtml = objChilkat.ConvertHtmlToXml(Responce); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion //Get User Id From page if (!(ProfilePageSource == null)) { if (ProfilePageSource.Contains("data-user-id=")) { try { String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(ProfilePageSource, "data-user-id="); if (getDataInArr.Count() > 0) { foreach (var item in getDataInArr) { if (item.Contains("<h1")) { int startindex = (1); int endindex = item.IndexOf(">") - 1; ; id = item.Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty); if (id.Contains("\n")) { string[] arr =System.Text.RegularExpressions.Regex.Split(id,"\""); id=arr[0]; } dataLst.Add("id", id); break; } } //whiteListedUserCount++; } } catch (Exception) { } } } //else //{ // blackListedUserCount++; //} int counterdata = 0; xBeginSearchAfter = null; string dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNameTruncated-link u-textInheritColor js-nav js-action-profile-name");//"profile-field"); while ((xNode != null)) { try { string str = xNode.GetXml(); xBeginSearchAfter = xNode; if (counterdata == 1) { website = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else if (counterdata == 0) { name = xNode.AccumulateTagContent("text", "script|style"); dataLst.Add("name", name); counterdata++; } else { break; } xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNameTruncated-link u-textInheritColor js-nav js-action-profile-name");//"profile-field"); } catch (Exception) { } } //Get Screen name and ID xBeginSearchAfter = null; //string dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "u-linkComplex-target"); while ((xNode != null)) { string str = xNode.GetXml(); xBeginSearchAfter = xNode; screen_name = xNode.AccumulateTagContent("text", "script|style").Replace("@", string.Empty); dataLst.Add("screen_name", screen_name); break; } xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//"location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; location = xNode.AccumulateTagContent("text", "script|style"); dataLst.Add("location", location); break; } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir"); while ((xNode != null)) { xBeginSearchAfter = xNode; description = xNode.AccumulateTagContent("text", "script|style"); dataLst.Add("description", description); break; } int counterData = 0; xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav"); xBeginSearchAfter = xNode; if (counterData == 0) { NoOfTweet = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet",string.Empty); dataLst.Add("statuses_count", NoOfTweet); counterData++; } //dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor"); xBeginSearchAfter = xNode; if (counterData == 1) { following_count = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty); dataLst.Add("friends_count", following_count); counterData++; } xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor"); //while ((xNode != null)) //{ // //xBeginSearchAfter = xNode; //if (counterData == 0) //{ // NoOfTweet = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet",string.Empty); // dataLst.Add("statuses_count", NoOfTweet); // counterData++; //} xBeginSearchAfter = xNode; if (counterData == 2) { followers_count = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty); dataLst.Add("followers_count", followers_count); counterData++; } //else //{ // break; //} xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//"js-nav"); // } status = "Ok"; //} //catch (Exception) //{ // status = ""; //} return dataLst; }
//public static int blackListedUserCount = 0; //public static int whiteListedUserCount = 0; public static string GetUserIDFromUsername(string username, out string Status) { string GetStatus = string.Empty; Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper(); clsDBQueryManager DB = new clsDBQueryManager(); DataSet ds = DB.GetUserId(username); string user_id = string.Empty; try { if (ds.Tables[0].Rows.Count > 0) { foreach (DataRow dataRow in ds.Tables["tb_UsernameDetails"].Rows) { user_id = dataRow.ItemArray[0].ToString(); Status = "No Error"; return user_id; } } } catch { }; try { #region gs httpHelper code //if (username.Contains("@")) //{ //} //string pagesource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?screen_name=" + username), "api.twitter.com", ""); #endregion string id = string.Empty; string pagesource = string.Empty; ChilkatHttpHelpr httpHelper1 = new ChilkatHttpHelpr(); if (NumberHelper.ValidateNumber(username)) { pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + username); if (string.IsNullOrEmpty(pagesource)) { pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + username); } } else { pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username); if (string.IsNullOrEmpty(pagesource)) { pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username); } } if (!pagesource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.") && !pagesource.Contains("Sorry, that page does not exist") && !pagesource.Contains("User has been suspended")) { if (NumberHelper.ValidateNumber(username)) { int length = pagesource.IndexOf("</screen_name>"); username = pagesource.Substring(pagesource.IndexOf("<screen_name>"), length - pagesource.IndexOf("<screen_name>")).Replace("<screen_name>", ""); user_id = username; GetStatus = "No Error"; } else { int length = pagesource.IndexOf("</id>"); id = pagesource.Substring(pagesource.IndexOf("<id>"), length - pagesource.IndexOf("<id>")).Replace("<id>", ""); user_id = id; GetStatus = "No Error"; } } else if (pagesource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.")) { GetStatus = "Rate limit exceeded"; } else if (pagesource.Contains("Sorry, that page does not exist")) { GetStatus = "Sorry, that page does not exist"; } else if (pagesource.Contains("User has been suspended")) { GetStatus = "User has been suspended"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserIDFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserIDFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); GetStatus = "Error"; } Status = GetStatus; return user_id; }
public static string GetPhotoFromUsername_New(string username) { //Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper(); ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr(); string user_id = string.Empty; string containsImage = "false"; try { string ImageLink = string.Empty; string PageSource = string.Empty; if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username)) { PageSource = httpHelper.GetHtml("https://twitter.com/account/redirect_by_id?id=" + username); } else { PageSource = httpHelper.GetHtml("https://twitter.com/" + username); } if (!string.IsNullOrEmpty(PageSource)) { try { int indexStart = PageSource.IndexOf("profile-picture media-thumbnail"); if (PageSource.Contains("profile-picture media-thumbnail")) { string start = PageSource.Substring(indexStart); int endIndex = start.IndexOf(" </a>"); string end = start.Substring(0, endIndex); if (!end.Contains(".png")) { int StartIndex = end.IndexOf("src="); string start1 = end.Substring(StartIndex); int EndIndex = start1.IndexOf(">"); string ImageLinkData = start1.Substring(0, EndIndex); ImageLink = System.Text.RegularExpressions.Regex.Split(ImageLinkData, "alt")[0].Replace("src=\"", string.Empty).Replace("\"", string.Empty); } else { ImageLink = string.Empty; } } if (!string.IsNullOrEmpty(ImageLink) && ImageLink.Contains("profile_images")) { containsImage = "true"; } else if (string.IsNullOrEmpty(ImageLink)) { containsImage = "false"; } else if (PageSource.Contains("Sorry, that page does not exist")) { containsImage = "false"; } else { containsImage = "false"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } } else if (PageSource.Contains("Clients may not make more than 150 requests per hour") && PageSource.Contains("Rate limit exceeded")) { containsImage = "Rate limit exceeded"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return containsImage; }
public List<string> GetFollowers(string userID, out string ReturnStatus) { string cursor = "-1"; string FollowingUrl = string.Empty; List<string> lstIds = new List<string>(); try { StartAgain: if (NumberHelper.ValidateNumber(userID)) { FollowingUrl = "https://api.twitter.com/1/followers/ids.json?cursor=" + cursor + "&id=" + userID + "";//"https://api.twitter.com/1/friends/ids.json?cursor=-1&screen_name=SocioPro"; } else { FollowingUrl = "https://api.twitter.com/1/followers/ids.json?cursor=" + cursor + "&screen_name=" + userID + "";//"https://api.twitter.com/1/friends/ids.json?cursor=-1&screen_name=SocioPro"; } #region gs http helper code //https://api.twitter.com/1/following/ids.json?cursor=-1&screen_name=SocioPro //string Data = globushttpHelper.getHtmlfromUrl(new Uri(FollowingUrl), "", ""); #endregion ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); string Data = HttpHelper.GetHtml(FollowingUrl); if (Data.Contains("401 Unauthorized")) { ReturnStatus = "Account is Suspended. "; return new List<string>(); } else if (!Data.Contains("Rate limit exceeded") && !Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}") && !string.IsNullOrEmpty(Data)) { int FirstPoint = Data.IndexOf("["); int SecondPoint = Data.IndexOf("]"); string FollowingIds = Data.Substring(FirstPoint, SecondPoint - FirstPoint).Replace("[", string.Empty).Replace("]", string.Empty); List<string> tempid = FollowingIds.Split(',').ToList(); foreach (string item in tempid) { lstIds.Add(item); } if (Data.Contains("next_cursor_str")) { int startindex = Data.IndexOf("\"next_cursor_str\":"); string start = Data.Substring(startindex).Replace("\"next_cursor_str\":", ""); int lastindex = start.IndexOf("\",\""); string end = start.Substring(0, lastindex).Replace("\"", ""); cursor = end; if (cursor != "0") { goto StartAgain; } } ReturnStatus = "No Error"; return lstIds; } else if (Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}")) { ReturnStatus = "Sorry, that page does not exist :" + userID; return lstIds; } else if (Data.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.")) { ReturnStatus = "Rate limit exceeded. Clients may not make more than 150 requests per hour.:-" + userID; return lstIds; } else { ReturnStatus = "Error"; return lstIds; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowers() -- " + userID + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowers() -- " + userID + " --> " + ex.Message, Globals.Path_TwtErrorLogs); ReturnStatus = "Error"; return lstIds; } }
public List <string> GetTweetData_Scrape(string keyword, out string returnStatus) { string status = string.Empty; List <string> lst_TweetIDs = new List <string>(); try { string user_id = string.Empty; #region old code //if (!NumberHelper.ValidateNumber(keyword)) //{ // user_id = GetUserIDFromUsername(keyword); //} //else //{ // user_id = keyword; //} //if (!string.IsNullOrEmpty(user_id)) //{ //screen_name #endregion string searchURL = string.Empty; if (NumberHelper.ValidateNumber(keyword)) { searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + keyword + "&count=" + TweetExtractCount; } else { searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&screen_name=" + keyword + "&count=" + TweetExtractCount; } //string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); string res_Get_searchURL = HttpHelper.GetHtml(searchURL); if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && !res_Get_searchURL.Contains("Not authorized") && res_Get_searchURL.Contains("created_at") && !string.IsNullOrEmpty(res_Get_searchURL)) { string[] splitRes = Regex.Split(res_Get_searchURL, "{\"created_at");//Regex.Split(res_Get_searchURL, "{\"created_at\""); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { //string text = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "<text>"); string Tweet = Globussoft.GlobusHttpHelper.ParseJson(item, "text");//Globussoft.GlobusHttpHelper.parseText(item); string data = keyword + ":" + Tweet; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(data, Globals.Path_TweetExtractor); lst_TweetIDs.Add(data); } status = "No Error"; } else if (res_Get_searchURL.Contains("Rate limit exceeded")) { status = "Rate limit exceeded"; } else if (res_Get_searchURL.Contains("Sorry, that page does not exist")) { status = "Sorry, that page does not exist"; } else if (res_Get_searchURL.Contains("Not authorized")) { status = "Not Authorized"; } else if (string.IsNullOrEmpty(res_Get_searchURL)) { status = "Not Authorized"; } else { status = "Empty"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); status = "Error"; } returnStatus = status; return(lst_TweetIDs); }
public List <string> GetRetweetData_Scrape(string keyword, out string returnStatus) { string status = string.Empty; List <string> lst_ReTweetIDs = new List <string>(); try { string searchURL = string.Empty; if (!NumberHelper.ValidateNumber(keyword)) { searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?screen_name=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true"; } else if (NumberHelper.ValidateNumber(keyword)) { searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?id=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true"; } //string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); string res_Get_searchURL = HttpHelper.GetHtml(searchURL); if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && res_Get_searchURL.Contains("created_at") && !string.IsNullOrEmpty(res_Get_searchURL)) { string[] splitRes = Regex.Split(res_Get_searchURL, "<status>");//Regex.Split(res_Get_searchURL, "{\"created_at\""); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { string Tweet = string.Empty; string Tweeter = string.Empty; try { int startIndex = item.IndexOf("<text>"); string start = item.Substring(startIndex); int endIndex = start.IndexOf("</text>"); string end = start.Substring(0, endIndex); Tweet = end.Replace("<text>", ""); int startOfInndex = Tweet.IndexOf(":"); Tweeter = Tweet.Substring(0, startOfInndex); Tweet = Tweet.Replace(Tweeter, ""); } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } string data = keyword + ":" + Tweeter + ":" + Tweet.Replace(":", "^"); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(data, Globals.Path_RETweetExtractor); lst_ReTweetIDs.Add(data); } } else if (res_Get_searchURL.Contains("Rate limit exceeded")) { status = "Rate limit exceeded"; } else if (res_Get_searchURL.Contains("Sorry, that page does not exist")) { status = "Sorry, that page does not exist"; } else if (res_Get_searchURL.Contains("Not authorized")) { status = "Not Authorized"; } else if (string.IsNullOrEmpty(res_Get_searchURL)) { status = "Not Authorized"; } else { status = "Empty"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); status = "Error"; } returnStatus = status; return(lst_ReTweetIDs); }
public static string GetPhotoFromUsername(string username) { //Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper(); ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr(); string user_id = string.Empty; string containsImage = "false"; try { string ImageLink = string.Empty; string PageSource = string.Empty; //http://a0.twimg.com/sticky/default_profile_images/default_profile_0_normal.png if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username)) { //PageSource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?user_id=" + username + "&include_entities=true"), "", ""); PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + username + "&include_entities=true"); } else { //PageSource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?screen_name=" + username + "&include_entities=true"), "", ""); PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username + "&include_entities=true"); } if (!string.IsNullOrEmpty(PageSource)) { try { int indexStart = PageSource.IndexOf("<profile_image_url>"); if (PageSource.Contains("<profile_image_url>")) { string start = PageSource.Substring(indexStart); int endIndex = start.IndexOf("</profile_image_url>"); string end = start.Substring(0, endIndex); ImageLink = end.Replace("<profile_image_url>", ""); } if (!string.IsNullOrEmpty(ImageLink) && ImageLink.Contains("/sticky/default_profile_images/default_profile")) { containsImage = "false"; } else if (PageSource.Contains("Sorry, that page does not exist")) { containsImage = "false"; } else { containsImage = "true"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } } else if (PageSource.Contains("Clients may not make more than 150 requests per hour") && PageSource.Contains("Rate limit exceeded")) { containsImage = "Rate limit exceeded"; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return containsImage; }
public void PostAddcompanyUrl(object parameter) { try { try { if (IsStop) { return; } if (!IsStop) { lstLinkedinFollowCompanyUrlThraed.Add(Thread.CurrentThread); lstLinkedinFollowCompanyUrlThraed.Distinct().ToList(); Thread.CurrentThread.IsBackground = true; } } catch { } if (!string.IsNullOrEmpty(txtNumberOfFollowPerAccount.Text) && NumberHelper.ValidateNumber(txtNumberOfFollowPerAccount.Text)) { LinkedinCompanyFollow.CountPerAccount = Convert.ToInt32(txtNumberOfFollowPerAccount.Text); } Array paramsArray = new object[1]; paramsArray = (Array)parameter; KeyValuePair <string, LinkedInMaster> item = (KeyValuePair <string, LinkedInMaster>)paramsArray.GetValue(0); LinkedinLogin Login = new LinkedinLogin(); Login.accountUser = item.Key; Login.accountPass = item.Value._Password; Login.proxyAddress = item.Value._ProxyAddress; Login.proxyPort = item.Value._ProxyPort; Login.proxyUserName = item.Value._ProxyUsername; Login.proxyPassword = item.Value._ProxyPassword; GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); ChilkatHttpHelpr ChilkatHttpHelper = new ChilkatHttpHelpr(); LinkedinCompanyFollow obj_FollowCompany = new LinkedinCompanyFollow(Login.accountUser, Login.accountPass, Login.proxyAddress, Login.proxyPort, Login.proxyUserName, Login.proxyPassword); Login.logger.addToLogger += new EventHandler(logger_SearchFollowCompanyaddToLogger); int minDelay = 20; int maxDelay = 25; if (!string.IsNullOrEmpty(txtFollowCompanyMinDelay.Text) && NumberHelper.ValidateNumber(txtFollowCompanyMinDelay.Text)) { minDelay = Convert.ToInt32(txtFollowCompanyMinDelay.Text); } if (!string.IsNullOrEmpty(txtFollowCompanyMaxDelay.Text) && NumberHelper.ValidateNumber(txtFollowCompanyMaxDelay.Text)) { maxDelay = Convert.ToInt32(txtFollowCompanyMaxDelay.Text); } if (!Login.IsLoggedIn) { Login.LoginHttpHelper(ref HttpHelper); //Login.LoginWithChilkatHelper(ref ChilkatHttpHelper); } if (Login.IsLoggedIn) { LinkedinCompanyFollow LinkdinFollowComp = new LinkedinCompanyFollow(); //LinkdinFollowComp.logger.addToLogger += new EventHandler(logger_SearchFollowCompanyaddToLogger); try { Dictionary <string, string> Result = new Dictionary <string, string>(); string MessagePosted = obj_FollowCompany.PostAddCompanyUsingUrl(ref HttpHelper, Login.accountUser, minDelay, maxDelay); } catch { } } } catch (Exception ex) { GlobusFileHelper.AppendStringToTextfileNewLine("DateTime :- " + DateTime.Now + " :: Error --> Manage Follow Company URL --> PostAddcompanyUrl() >>>> " + ex.Message + "StackTrace --> >>>" + ex.StackTrace, Globals.Path_LinkedinErrorLogs); GlobusFileHelper.AppendStringToTextfileNewLine("DateTime :- " + DateTime.Now + " :: Error --> Manage Follow Company URL --> PostAddcompanyUrl() >>>> " + ex.Message + "StackTrace --> >>>" + ex.StackTrace, Globals.Path_LinkedinAddSearchGroupErrorLogs); } finally { counter_AddFollowUrl--; if (counter_AddFollowUrl == 0) { btnFollowCompany.Invoke(new MethodInvoker(delegate { AddLoggerFollowCompanyUrl("[ " + DateTime.Now + " ] => [ PROCESS COMPLETED ]"); AddLoggerFollowCompanyUrl("----------------------------------------------------------------------------------------------------------"); btnFollowCompany.Cursor = Cursors.Default; })); } } }
public List<string> GetFollowings(string userID, out string ReturnStatus) { try { string cursor = "-1"; string FollowingUrl = string.Empty; StartAgain: if (NumberHelper.ValidateNumber(userID)) { FollowingUrl = "https://api.twitter.com/1/following/ids.json?cursor=" + cursor + "&id=" + userID + "";//"https://api.twitter.com/1/following/ids.json?cursor=-1&screen_name=SocioPro"; } else { FollowingUrl = "https://api.twitter.com/1/following/ids.json?cursor=" + cursor + "&screen_name=" + userID + "";//"https://api.twitter.com/1/following/ids.json?cursor=-1&screen_name=SocioPro"; } ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); string Data = HttpHelper.GetHtml(FollowingUrl);//.getHtmlfromUrl(new Uri(FollowingUrl), "", ""); if (Data.Contains("that page does not exist")) { ReturnStatus = "that page does not exist"; return new List<string>(); } else if (!Data.Contains("Rate limit exceeded")) { int FirstPoint = Data.IndexOf("["); int SecondPoint = Data.IndexOf("]"); string FollowingIds = Data.Substring(FirstPoint, SecondPoint - FirstPoint).Replace("[", string.Empty).Replace("]", string.Empty); List<string> lstIds = FollowingIds.Split(',').ToList(); if (Data.Contains("next_cursor_str")) { int startindex = Data.IndexOf("\"next_cursor_str\":"); string start = Data.Substring(startindex).Replace("\"next_cursor_str\":", ""); int lastindex = start.IndexOf("\",\""); string end = start.Substring(0, lastindex).Replace("\"", ""); cursor = end; if (cursor != "0") { goto StartAgain; } } ReturnStatus = "No Error"; return lstIds; } else { ReturnStatus = "Error"; return new List<string>(); } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings() -- " + userID + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings() -- " + userID + " --> " + ex.Message, Globals.Path_TwtErrorLogs); ReturnStatus = "Error"; return new List<string>(); } }
public static void RequestJSCSSIMG(string pageSource, ref Chilkat.Http http) { ServicePointManager.ServerCertificateValidationCallback = delegate { return(true); }; List <string> listURLs = new List <string>(); ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); //CSS Request foreach (string item in GetHrefsFromString(pageSource)) { if (item.Contains(".css")) { string cssSource = item.Replace(" ", "").Trim(); try { //string res = HttpHelper.GetHtmlProxy(cssSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); listURLs.Add(cssSource); } catch (Exception) { Thread.Sleep(500); try { string res = HttpHelper.GetHtmlProxy(cssSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); } catch (Exception) { } } } } //JS Request string[] scriptArr = Regex.Split(pageSource, "/script>"); foreach (string item in scriptArr) { try { if (item.Contains("static.ak.") || item.Contains("profile.ak.")) { int startIndx = item.LastIndexOf("src=") + "src=".Length + 1; int endIndx = item.IndexOf(">", startIndx) - 1; string jsSource = item.Substring(startIndx, endIndx - startIndx); //if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.")) if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.") || jsSource.StartsWith("http://profile.ak.") || jsSource.StartsWith("https://profile.ak.") || jsSource.StartsWith("http://s-profile.ak.") || jsSource.StartsWith("https://s-profile.ak.")) { try { //string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); listURLs.Add(jsSource); } catch (Exception) { Thread.Sleep(500); try { string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); } catch (Exception) { } } } } } catch (Exception) { } } string[] moreScriptArray = Regex.Split(pageSource, "\"src\":"); foreach (string item in moreScriptArray) { try { int startIndx = 1; int endIndx = item.IndexOf("\"", startIndx); string jsSource = item.Substring(startIndx, endIndx - startIndx).Replace("\\", ""); if (jsSource.Contains(".js")) { //string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); listURLs.Add(jsSource); } } catch (Exception) { } } ///IMG Request string[] imageArr = Regex.Split(pageSource, "<img"); foreach (string item in imageArr) { try { if (item.Contains("static.ak.") || item.Contains("profile.ak.")) { int startIndx = item.IndexOf("src=") + "src=".Length + 1; int endIndx = item.IndexOf("\"", startIndx + 1); string jsSource = item.Substring(startIndx, endIndx - startIndx); //if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.")) if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.") || jsSource.StartsWith("http://profile.ak.") || jsSource.StartsWith("https://profile.ak.") || jsSource.StartsWith("http://s-profile.ak.") || jsSource.StartsWith("https://s-profile.ak.")) { if (jsSource.Contains(".png") || jsSource.Contains(".gif") || jsSource.Contains(".jpg") || jsSource.Contains(".jpeg")) { try { //string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); listURLs.Add(jsSource); } catch (Exception) { Thread.Sleep(500); try { string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); } catch (Exception) { } } } } } } catch { }; } listURLs = listURLs.Distinct().ToList(); foreach (string item in listURLs) { try { string res = HttpHelper.GetHtmlProxy(item, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http); } catch { }; } }
//public List<string> GetFollowings_New(string userID, out string ReturnStatus, ref Globussoft.GlobusHttpHelper HttpHelper) //{ // Log("[ " + DateTime.Now + " ] => [ Searching For Following For " + userID + " ]"); // string cursor = "-1"; // string FollowingUrl = string.Empty; // List<string> lstIds = new List<string>(); // try // { // StartAgain: // string Data = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + userID + "/following/users?cursor=" + cursor + "&include_available_features=1&include_entities=1&is_forward=true"), "", ""); // if (string.IsNullOrEmpty(Data)) // { // for (int i = 1; i <= 3; i++) // { // Thread.Sleep(3000); // Data = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + userID + "/followers/users?cursor=" + cursor + "&include_available_features=1&include_entities=1&is_forward=true"), "", ""); // if (!string.IsNullOrEmpty(Data)) // { // break; // } // } // } // if (Data.Contains("401 Unauthorized")) // { // ReturnStatus = "Account is Suspended. "; // return lstIds; // } // else if (!Data.Contains("Rate limit exceeded") && !Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}") && !string.IsNullOrEmpty(Data)) // { // //string[] arraydata = Regex.Split(Data, "js-stream-item stream-item stream-item"); // string[] arraydata; // string startWitn = string.Empty; // if (Data.Contains("js-stream-item stream-item stream-item")) // { // arraydata = Regex.Split(Data, "js-stream-item stream-item stream-item"); // startWitn = ""; // } // else // { // arraydata = Regex.Split(Data, "js-stream-item"); // startWitn = "\\\" role=\\\"listitem\\\""; // } // arraydata = arraydata.Skip(1).ToArray(); // foreach (string id in arraydata) // { // string userid = string.Empty; // string username = string.Empty; // if (!id.StartsWith(startWitn)) // { // continue; // } // try // { // int startindex = id.IndexOf("data-item-id=\\\""); // string start = id.Substring(startindex).Replace("data-item-id=\\\"", ""); // int endindex = start.IndexOf("\\\""); // string end = start.Substring(0, endindex); // userid = end; // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings_New() -- " + userID + " --> userid --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings_New() -- " + userID + " --> userid --> " + ex.Message, Globals.Path_TwtErrorLogs); // } // try // { // int startindex = id.IndexOf("data-screen-name=\\\""); // string start = id.Substring(startindex).Replace("data-screen-name=\\\"", ""); // int endindex = start.IndexOf("\\\""); // string end = start.Substring(0, endindex); // username = end; // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings_New() -- " + userID + " --> username --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings_New() -- " + userID + " --> username --> " + ex.Message, Globals.Path_TwtErrorLogs); // } // if (CounterDataNo > 0) // { // if (lstIds.Count == CounterDataNo) // { // ReturnStatus = "No Error"; // lstIds = lstIds.Distinct().ToList(); // return lstIds; // } // else // { // lstIds.Add(userid + ":" + username); // Globals.lstScrapedUserIDs.Add(userid); // Log("[ " + DateTime.Now + " ] => [ " + username + ":" + userid + " ]"); // GlobusFileHelper.AppendStringToTextfileNewLine(userID + "," + userid + "," + username, Globals.Path_ScrapedFollowingsList); // } // } // try // { // //if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(userid)) // //{ // // string query = "INSERT INTO tb_UsernameDetails (Username , Userid) VALUES ('" + username + "' ,'" + userid + "') "; // // DataBaseHandler.InsertQuery(query, "tb_UsernameDetails"); // //} // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings_New() -- " + userID + " --> Database --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings_New() -- " + userID + " --> DataBase --> " + ex.Message, Globals.Path_TwtErrorLogs); // } // } // if (Data.Contains("\"has_more_items\":true")) // { // int startindex = Data.IndexOf("\"cursor\":"); // string start = Data.Substring(startindex).Replace("\"cursor\":", ""); // int lastindex = start.IndexOf("\",\""); // if (lastindex < 0) // { // lastindex = start.IndexOf("\"}"); // } // string end = start.Substring(0, lastindex).Replace("\"", ""); // cursor = end; // if (cursor != "0") // { // if (NoOfFollowingsToBeunfollowed > 0) // { // lstIds = lstIds.Distinct().ToList(); // //commented by prabhat // //if (NoOfFollowingsToBeunfollowed == lst_structTweetIDs.Count) // if (NoOfFollowingsToBeunfollowed == lstIds.Count) // { // ReturnStatus = "No Error"; // return lstIds; // } // } // goto StartAgain; // } // } // ReturnStatus = "No Error"; // lstIds = lstIds.Distinct().ToList(); // return lstIds; // } // else if (Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}")) // { // ReturnStatus = "Sorry, that page does not exist :" + userID; // lstIds = lstIds.Distinct().ToList(); // return lstIds; // } // else if (Data.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.")) // { // ReturnStatus = "Rate limit exceeded. Clients may not make more than 150 requests per hour.:-" + userID; // lstIds = lstIds.Distinct().ToList(); // return lstIds; // } // else // { // ReturnStatus = "Error"; // lstIds = lstIds.Distinct().ToList(); // return lstIds; // } // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings_New() -- " + userID + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings_New() -- " + userID + " --> " + ex.Message, Globals.Path_TwtErrorLogs); // ReturnStatus = "Error"; // lstIds = lstIds.Distinct().ToList(); // return lstIds; // } //} public List<string> GetHashTags(out string returnStatus) { List<string> HashTags = new List<string>(); try { //string pagesource = globushttpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/trends/daily.json"), "", ""); ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); string pagesource = HttpHelper.GetHtml("https://api.twitter.com/1/trends/daily.json"); if (!pagesource.Contains("Rate limit exceeded")) { string[] array = Regex.Split(pagesource, "\"name\":"); array = array.Skip(1).ToArray(); foreach (string item in array) { try { int startindex = item.IndexOf("\""); string Start = item.Substring(startindex); int endIndex = Start.IndexOf("\","); string End = Start.Substring(0, endIndex).Replace("\"", ""); string HashTag = End; if (HashTag.Contains("#")) { HashTags.Add(HashTag); } } catch (Exception ex) { returnStatus = "Error"; return new List<string>(); } } returnStatus = "No Error"; return HashTags; } else { returnStatus = "Error"; return new List<string>(); } } catch (Exception ex) { returnStatus = "Error"; return new List<string>(); } }
public void DataParser(string response, ref FacebookUser fbUser) { try { try { string Response = response; ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr(); string FirstPgaeName = string.Empty; string SecondPgaeName = string.Empty; string otherPgaeName = string.Empty; string PageTitle = string.Empty; string PageTitleLink = string.Empty; string SourceTitle = string.Empty; string SourceTitleLink = string.Empty; string memberCount = string.Empty; string FirstPgaeNameLink = string.Empty; string SecondPgaeNameLink = string.Empty; string otherPgaeNameLink = string.Empty; string likerData = string.Empty; string SecondlikerData = string.Empty; string ThirdlikerData = string.Empty; string AboutData = string.Empty; string AboutDataLink = string.Empty; bool MemberPage = false; string[] DataList = System.Text.RegularExpressions.Regex.Split(Response, "<div class=\""); foreach (string item2 in DataList) { #region Title if (item2.Contains(""title"")) { List <string> hrefLink = objChilkat.GetHrefFromString(item2); try { string titleData = item2.Substring(item2.IndexOf("\"><a") + 5); string DataPage = GetSbstringData(titleData, "\">", "<"); PageTitle = DataPage; PageTitleLink = hrefLink[0].ToString(); } catch { }; } #endregion #region GroupSource if (item2.Contains("quot;sub_headers"")) { List <string> hrefLink = objChilkat.GetHrefFromString(item2); try { string titleData = item2.Substring(item2.IndexOf("\"><a") + 5); string DataPage = GetSbstringData(titleData, "\">", "<"); SourceTitle = DataPage; if (hrefLink.Count() > 0) { SourceTitleLink = hrefLink[0].ToString(); } } catch { }; } #endregion # region Members if (item2.Contains("members") && item2.Count() < 150 && MemberPage == false) { MemberPage = true; try { string MemberData = GetSbstringData(item2, "\">", "<").Replace("members", "").Replace(",", ""); memberCount = MemberData; } catch { }; } #endregion #region Aboutdata #endregion #region Friendslink #endregion } string Username = string.Empty; if (!string.IsNullOrEmpty(PageTitle)) { string data = PageTitleLink.Substring(0, PageTitleLink.IndexOf("?")); Username = data.Split('/')[3]; // Username = PageTitle + " : " + PageTitleLink; } string Likes = string.Empty; if (!string.IsNullOrEmpty(FirstPgaeName)) { //Likes ="Likes "+ FirstPgaeName + " : " + FirstPgaeNameLink; Likes = "Likes " + FirstPgaeNameLink; } if (!string.IsNullOrEmpty(SecondPgaeName)) { // Likes = Likes + " and " + SecondPgaeName + " : " + SecondPgaeNameLink; Likes = Likes + " and " + SecondPgaeNameLink; } if (!string.IsNullOrEmpty(otherPgaeName)) { // Likes = Likes + " and " + otherPgaeName + " : " + otherPgaeNameLink; Likes = Likes + " and " + otherPgaeNameLink; } if (Username.Count() > 5) { try { string FollowerData = string.Empty; string Userdata = Username.Replace(",", "") + "," + PageTitle.Replace(",", "") + "," + SourceTitle.Replace(",", "") + "," + memberCount.Replace(",", "") + "," + AboutData.Replace(",", ""); string FileData = Environment.GetFolderPath(Environment.SpecialFolder.Desktop) + "\\FBScraperDataDirectory\\SearchUrlUserId.txt"; if (!File.Exists(FileData)) { try { string DataHeader = " UserId " + "\t" + " Likes " + "\t" + " Studied In " + " \\t " + " Listens " + " \\t " + "Followers" + "\\t " + "Watches"; GlobusFileHelper.AppendStringToTextfileNewLine(DataHeader, exportFilePathAccountVerification + "\\GroupSearchData"); } catch { }; } GlobusFileHelper.AppendStringToTextfileNewLine(Userdata, exportFilePathAccountVerification + "\\GroupSearchData"); //AppendStringToTextFileNewLine(Data, FileData); } catch { }; } } catch { }; }
/// <summary> /// Checks the status of the CPUID from Database /// If status is Active, MainFrm starts /// </summary> public bool ValidateCPUID(ref string statusMessage, string servr, ref string username, ref string Password, ref string txnID, string freeTrialKey, string cpuID, ref string Email) { //string cpuID = getCPUID(); try { #region Drct //string cpuID = FetchMacId(); //string SelectQuery = "Select * from users where cpuid='" + cpuID + "'"; //DataSet ds = DataBaseHandler.SelectQuery(SelectQuery, "users"); //if (ds.Tables[0].Rows.Count == 1) //{ // string status = ds.Tables[0].Rows[0]["status"].ToString(); // if (status.ToLower() == "active") // { // statusMessage = "active"; // return true; // } // else if (status.ToLower() == "nonactive") // { // statusMessage = "nonactive"; // return false; // } // else if (status.ToLower() == "suspended") // { // statusMessage = "suspended"; // return false; // } //} #endregion #region Through php //string cpuID = FetchMacId(); //ChilkatHttpHelpr HttpHelpr = new ChilkatHttpHelpr(); HttpHelpr = new ChilkatHttpHelpr(); #region Servr 1 { string res = string.Empty; res = HttpHelpr.GetHtml("http://" + servr + "/GetUserData.php?cpid=" + cpuID + ""); if (string.IsNullOrEmpty(res)) { System.Threading.Thread.Sleep(1000); res = HttpHelpr.GetHtml("http://" + servr + "/releases/GetUserData.php?cpid=" + cpuID + ""); } if (!string.IsNullOrEmpty(res)) { string activationstatus = string.Empty; string dateTime = string.Empty; string trimmed_response = res.Replace("<pre>", "").Replace("</pre>", "").Trim().ToLower(); string[] array_status = System.Text.RegularExpressions.Regex.Split(trimmed_response, "<:>"); try { activationstatus = array_status[0].ToLower(); } catch { } try { dateTime = array_status[1].ToLower(); } catch { } try { username = array_status[2].ToLower(); } catch { } try { Password = array_status[3].ToLower(); } catch { }; try { txnID = array_status[4].ToLower(); } catch { } try { Email = array_status[5].ToLower(); } catch { }; try { Globals.licType = array_status[6].ToLower(); } catch { }; try { if (txnID.Contains("freetrial")) { Globals.Licence_Details = username + ":" + dateTime + "&" + "FreeTrial" + ""; } else { Globals.Licence_Details = username + ":" + dateTime + "&" + "Full Version" + ""; } } catch { } if (trimmed_response.ToLower().Contains(freeTrialKey) && ((activationstatus.ToLower() == "active") || (activationstatus.ToLower() == "nonactive"))) { PDGlobals.Instance.isfreeversion = true; if (CheckActivationUpdateStatus(cpuID, dateTime, activationstatus, servr)) { statusMessage = "active"; return(true); } else { statusMessage = "trialexpired"; return(false); } if (activationstatus.ToLower() == "active") { statusMessage = "active"; return(true); } else//else if (activationstatus.ToLower() == "nonactive") { //Update status as Active string updateRes = HttpHelpr.GetHtml("http://" + servr + "/UpdateStatus.php?cpid=" + cpuID + "&status=" + "Active"); if (string.IsNullOrEmpty(updateRes)) { System.Threading.Thread.Sleep(1000); updateRes = HttpHelpr.GetHtml("http://" + servr + "/UpdateStatus.php?cpid=" + cpuID + "&status=" + "Active"); } MessageBox.Show("Your Free Version is Activated"); return(true); } } else if (activationstatus.ToLower() == "active") { statusMessage = "active"; return(true); // DisableControls(); } else if (activationstatus.ToLower() == "nonactive") { statusMessage = "nonactive"; MessageBox.Show("Verification of your txn is under process.\n Please wait for your Transaction to be verified.\n Please Contact To Support Team to activate your license, Skype Id Is :- Facedominatorsupport"); return(false); //DisableControls(); } else if (trimmed_response.Contains("trialexpired")) { statusMessage = "trialexpired"; MessageBox.Show("Your 3 Days Trial Version has Expired. Please visit our site: facedominator.com to purchase your License"); return(false); } else if (trimmed_response.ToLower() == "suspended") { statusMessage = "suspended"; return(false); } else if (trimmed_response.Contains("no record found")) { statusMessage = "norecordfound"; return(false); } else { statusMessage = "Some Error in Licensing Server"; return(false); } } else { statusMessage = "ServerDown"; return(false); } } #endregion #endregion } catch (Exception ex) { Console.WriteLine(ex.StackTrace); statusMessage = "Error in License Validation"; MessageBox.Show(ex.StackTrace); } return(false); }
private void ScrapeWhotoFollow(TweetAccountManager tweetAccountManager, string Keyword, int NoOfPages) { try { AddThreadToDictionary(strModule(Module.WhoToScrap), tweetAccountManager.Username); int counter = 0; int PageCount = 1; // int NoOfData = NoOfPages * 20; List<string> username = new List<string>(); AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Getting Users To Scrape Data For ]"); while (counter < NoOfPages) { // string pagsource = tweetAccountManager.globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/who_to_follow/suggestions/search/users?q=" + Keyword + "&cursor=" + PageCount + "&include_available_features=1&include_entities=1&is_forward=true"), "", ""); string pagsource = ""; if (PageCount == 1) { pagsource = tweetAccountManager.globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/search/timeline?q=" + Keyword + "&cursor=" + PageCount + "&include_available_features=1&include_entities=1&is_forward=true"), "", ""); // https://twitter.com/who_to_follow/suggestions/search/users?q=software&cursor=1&include_available_features=1&include_entities=1&is_forward=true } else { //if (pagsource.Contains("\"has_more_items\":true")) string uri = "https://twitter.com/i/search/timeline?q=" + Keyword + "&mode=users&include_available_features=1&include_entities=1&last_note_ts=555&scroll_cursor=USER-0-" + PageCount * 20; pagsource = tweetAccountManager.globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/search/timeline?q=" + Keyword + "&mode=users&include_available_features=1&include_entities=1&last_note_ts=555&scroll_cursor=USER-0-" + PageCount * 20), "", ""); } // if (pagsource.Contains("has-more-items")) if (pagsource.Contains("has_more_items")) { PageCount++; string[] Aray = Regex.Split(pagsource, "js-stream-item stream-item"); //string[] Aray = Regex.Split(pagsource, "js-stream-item stream-item stream-user-item"); //js-stream-item stream-item // string[] Aray = Regex.Split(pagsource, "fullname js-action-profile-name"); Aray = Aray.Skip(1).ToArray(); foreach (string item in Aray) { string Userid = string.Empty; string Username = string.Empty; try { //int startindex = item.IndexOf("=\\\""); int startindex = item.IndexOf("data-user-id=\""); int startindexForItem = item.IndexOf("data-item-id=\\\""); if (startindex >= 0 && PageCount==2) { string start = item.Substring(startindex).Replace("data-user-id=\"", ""); //string start = item.Substring(startindex).Replace("=\\\"", ""); //int endindex = start.IndexOf("\\\""); int endindex = start.IndexOf("\""); string end = start.Substring(0, endindex); Userid = end; } else if (startindexForItem >= 0) { string start = item.Substring(startindexForItem).Replace("data-item-id=\\\"", ""); //string start = item.Substring(startindex).Replace("=\\\"", ""); //int endindex = start.IndexOf("\\\""); int endindex = start.IndexOf("\\\""); string end = start.Substring(0, endindex); Userid = end; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_ScrapeUsersErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_TwtErrorLogs); } try { //int startindex = item.IndexOf("data-screen-name=\\\""); int startindex = item.IndexOf("data-screen-name=\""); int startindex2 = item.IndexOf("data-screen-name=\\\""); if (startindex >= 0) { //string start = item.Substring(startindex).Replace("data-screen-name=\\\"", ""); string start = item.Substring(startindex).Replace("data-screen-name=\"", ""); //int endindex = start.IndexOf("\\\""); int endindex = start.IndexOf("\""); string end = start.Substring(0, endindex); Username = end; username.Add(end); } if (startindex2 >= 0) { //string start = item.Substring(startindex).Replace("data-screen-name=\\\"", ""); string start = item.Substring(startindex2).Replace("data-screen-name=\\\"", ""); //int endindex = start.IndexOf("\\\""); int endindex = start.IndexOf("\\\""); string end = start.Substring(0, endindex); Username = end; username.Add(end); } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_ScrapeUsersErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_TwtErrorLogs); } try { if (!string.IsNullOrEmpty(Username) && !string.IsNullOrEmpty(Userid)) { string query = "INSERT INTO tb_UsernameDetails (Username , Userid) VALUES ('" + Username + "' ,'" + Userid + "') "; DataBaseHandler.InsertQuery(query, "tb_UsernameDetails"); } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() --> Database --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() --> DataBase --> " + ex.Message, Globals.Path_TwtErrorLogs); } AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + Username + " :: " + Userid + " ]"); } username = username.Distinct().ToList(); counter++; } else { AddToScrapeLogs("[ " + DateTime.Now + " ] => [ No More Pages To Scrape For Keyword : " + Keyword + " ]"); break; } } if (!File.Exists(Globals.Path_KeywordScrapedListData + ".csv")) { GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NOOFTWEETS , FOLLOWING , FOLLOWERS ", Globals.Path_KeywordScrapedListData + ".csv"); } foreach (string UserIds in username) { try { string ProfileName = string.Empty; string Location = string.Empty; string Bio = string.Empty; string website = string.Empty; string NoOfTweets = string.Empty; string Followers = string.Empty; string Followings = string.Empty; string userids = string.Empty; string TweetUsername = string.Empty; string Userid = string.Empty; string Username = string.Empty; ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr(); GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + UserIds), "", ""); string Responce = ProfilePageSource; #region Convert HTML to XML string xHtml = objChilkat.ConvertHtmlToXml(Responce); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion //xNode = xml.SearchForAttribute(xBeginSearchAfter, "ul", "class", "stats js-mini-profile-stats"); //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field"); //userids = xNode.GetAttrValue("data-user-id"); //userids = xNode.AccumulateTagContent("text", "script|style"); if (Responce.Contains("has-more-items")) { PageCount++; //string[] Aray = Regex.Split(Responce, "js-stream-item stream-item stream-item"); //Aray = Aray.Skip(1).ToArray(); //foreach (string item in Aray) { try { #region commented //int startindex = item.IndexOf("=\\\""); //int startindex = item.IndexOf("data-user-id="); //if (startindex >= 0) //{ // string start = item.Substring(startindex).Replace("data-user-id=\"", ""); // //string start = item.Substring(startindex).Replace("=\\\"", ""); // //int endindex = start.IndexOf("\\\""); // int endindex = start.IndexOf("\""); // string end = start.Substring(0, endindex); // userids = end; // break; //} #endregion int startindex = Responce.IndexOf("profile_id"); string start = Responce.Substring(startindex).Replace("profile_id", ""); int endindex = start.IndexOf(","); string end = start.Substring(0, endindex).Replace(""", "").Replace("\"", "").Replace(":", "").Trim(); userids = end.Trim(); TweetUsername = UserIds; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_ScrapeUsersErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_TwtErrorLogs); } } } int counterdata = 0; xBeginSearchAfter = null; string dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterdata == 0) { ProfileName = xNode.AccumulateTagContent("text", "script|style").Replace("Verified account", ""); counterdata++; } else if (counterdata == 1) { website = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else { break; } //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor"); } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "screen-name"); //while ((xNode != null)) //{ // xBeginSearchAfter = xNode; // TweetUsername = xNode.AccumulateTagContent("text", "script|style"); // break; //} xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim(); break; } xBeginSearchAfter = null; dataDescription = string.Empty; //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "location profile-field"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir"); while ((xNode != null)) { xBeginSearchAfter = xNode; Location = xNode.AccumulateTagContent("text", "script|style"); break; } int counterData = 0; xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim(); break; } xBeginSearchAfter = null; dataDescription = string.Empty; //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "location profile-field"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Location = xNode.AccumulateTagContent("text", "script|style"); break; } xBeginSearchAfter = null; dataDescription = string.Empty; //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "data-element-term", "tweet_stats"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterData == 0) { NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty); counterData++; } else if (counterData == 1) { Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty); counterData++; } else if (counterData == 2) { Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty); counterData++; } else { break; } //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor"); } if (!string.IsNullOrEmpty(userids)) { lock (WhoTofollowThreadLock) { GlobusFileHelper.AppendStringToTextfileNewLine(userids + "," + TweetUsername + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + ".csv"); } AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + userids + "," + TweetUsername + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]"); } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 3--> " + ex.Message, Globals.Path_ScrapeUsersErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 3--> " + ex.Message, Globals.Path_TwtErrorLogs); } } AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Finished Scraping For " + tweetAccountManager.Username + " ]"); AddToProxysLogs("-----------------------------------------------------------------------------------------------------------------------"); } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 2--> " + ex.Message, Globals.Path_ScrapeUsersErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 2--> " + ex.Message, Globals.Path_TwtErrorLogs); } }
public bool CheckPvtIP() { try { int Working = 0; string LoggedInIp = string.Empty; ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); GlobusHttpHelper HttpHelper1 = new GlobusHttpHelper(); //string pageSource = HttpHelper.getHtmlfromUrlIPChecker("https://twitter.com/", IPAddress, IPPort, IPUsername, IPpassword); string pageSource = HttpHelper1.getHtmlfromUrlIPChecker(new Uri("https://twitter.com/"), IPAddress, int.Parse(IPPort), IPUsername, IPpassword); if (string.IsNullOrEmpty(pageSource))//(string.IsNullOrEmpty(pageSource) && string.IsNullOrEmpty(PgSrcHome)) { Thread.Sleep(500); pageSource = HttpHelper1.getHtmlfromUrlIPChecker(new Uri("https://twitter.com/"), IPAddress, int.Parse(IPPort), IPUsername, IPpassword); if (string.IsNullOrEmpty(pageSource)) { return false; } } ///Logic to check... //if (pageSource.Contains("class=\"signin\"") && pageSource.Contains("class=\"signup\"") && pageSource.Contains("Twitter")) if (pageSource.Contains("Sign in") || pageSource.Contains("Sign up") && pageSource.Contains("Twitter")) { try { using (SQLiteConnection con = new SQLiteConnection(DataBaseHandler.CONstr)) { using (SQLiteDataAdapter ad = new SQLiteDataAdapter()) { Working = 1; string query = "Select * from tb_IP WHERE IPAddress = '" + IPAddress + "' AND IPPort = '" + IPPort + "' AND IPUsername = '******' AND IPpassword = '******' "; System.Data.DataSet ds = DataBaseHandler.SelectQuery(query, "tb_IP"); if (ds.Tables["tb_IP"].Rows.Count > 0) { string UpdateQuery = "UPDATE tb_IP SET Working = 1 WHERE IPAddress = '" + IPAddress + "' AND IPPort = '" + IPPort + "' AND IPUsername = '******' AND IPpassword = '******' "; DataBaseHandler.UpdateQuery(UpdateQuery, "tb_IP"); } else { string UpdateQuery = "INSERT INTO tb_IP VALUES ('" + IPAddress + "', '" + IPPort + "', '" + IPUsername + "','" + IPpassword + "', 1 , 1 , '" + "" + "' ) "; DataBaseHandler.InsertQuery(UpdateQuery, "tb_IP"); } Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(IPAddress + ":" + IPPort + ":" + IPUsername + ":" +IPpassword, Globals.Path_ExsistingPvtProxies); } } } catch (Exception ex) { GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> CheckPvtIP() 1 --> " + ex.Message, Globals.Path_IPSettingErroLog); GlobusFileHelper.AppendStringToTextfileNewLine("Error --> CheckPvtIP() 1 --> " + ex.Message, Globals.Path_TwtErrorLogs); } return true; } return false; } catch (Exception ex) { GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> CheckPvtIP() 2 --> " + ex.Message, Globals.Path_IPSettingErroLog); GlobusFileHelper.AppendStringToTextfileNewLine("Error --> CheckPvtIP() 2--> " + ex.Message, Globals.Path_TwtErrorLogs); return false; } }
public static List <string> GetAllFriends(ref ChilkatHttpHelpr chilkatHttpHelper, string userId) { List <string> finalList_Friends = new List <string>(); List <string> list_finalFriendsID = new List <string>(); try { string pgSource_Friends = chilkatHttpHelper.GetHtml("http://www.facebook.com/profile.php?id=" + userId + "&sk=friends"); string[] Friends = Regex.Split(pgSource_Friends, "user.php"); ParseFriendIDs(Friends, finalList_Friends); List <string> lstnewfriendid = new List <string>(); // {"collection_token":"1220529617:2356318349:2","cursor":"MDpub3Rfc3RydWN0dXJlZDoxMDAwMDA5MjM1MTg5MjY=","tab_key":"friends","profile_id":1220529617,"overview":false,"ftid":null,"order":null,"sk":"friends"} string collection_token = ""; string cursor = ""; //check if all friends loaded string patternAllFriendsLoaded = "\"TimelineAppCollection\",\"setFullyLoaded\",[],[\"pagelet_timeline_app_collection_"; int tempCount_AjaxRequests = 0; do // { try { collection_token = ""; cursor = ""; string[] arry = Regex.Split(pgSource_Friends, "enableContentLoader"); if (arry.Length > 1) { try { string rawData = arry[1]; int startIndx_collection_token = rawData.IndexOf("pagelet_timeline_app_collection_") + "pagelet_timeline_app_collection_".Length; int endIndx_collection_token = rawData.IndexOf("\"", startIndx_collection_token); collection_token = rawData.Substring(startIndx_collection_token, endIndx_collection_token - startIndx_collection_token); int startIndx_cursor = rawData.IndexOf(",\"", endIndx_collection_token) + ",\"".Length; int endIndx_cursor = rawData.IndexOf("\"", startIndx_cursor); cursor = rawData.Substring(startIndx_cursor, endIndx_cursor - startIndx_cursor); } catch (Exception ex) { GlobusLogHelper.log.Error("Error : " + ex.StackTrace); } } string raw_data = "{\"collection_token\":\"" + collection_token + "\",\"cursor\":\"" + cursor + "\",\"tab_key\":\"friends\",\"profile_id\":" + userId + ",\"overview\":false,\"ftid\":null,\"order\":null,\"sk\":\"friends\"}"; string encoded_raw_data = Uri.EscapeDataString(raw_data); string getURL_MoreFriendsAjax = "https://www.facebook.com/ajax/pagelet/generic.php/AllFriendsAppCollectionPagelet?data=" + encoded_raw_data + "&__user="******"&__a=1&__dyn=7n8ahyj2qmudwNAEU&__req=2"; string res_getURL_MoreFriendsAjax = chilkatHttpHelper.GetHtml(getURL_MoreFriendsAjax); pgSource_Friends = res_getURL_MoreFriendsAjax; string[] arry_UserData = Regex.Split(pgSource_Friends, "user.php"); ParseFriendIDs(arry_UserData, finalList_Friends); tempCount_AjaxRequests++; } catch (Exception ex) { GlobusLogHelper.log.Error("Error : " + ex.StackTrace); } } while ((collection_token != "" && cursor != "") || tempCount_AjaxRequests < 15 || pgSource_Friends.Contains(patternAllFriendsLoaded)); finalList_Friends.ForEach(delegate(String friendID) { if (friendID.Contains("&")) { friendID = friendID.Remove(friendID.IndexOf("&")); } list_finalFriendsID.Add(friendID); }); list_finalFriendsID = list_finalFriendsID.Distinct().ToList(); } catch (Exception ex) { GlobusLogHelper.log.Error("Error : " + ex.StackTrace); } return(list_finalFriendsID); }
public static void RequestJSCSSIMG(string pageSource, ref Chilkat.Http http) { ServicePointManager.ServerCertificateValidationCallback = delegate { return true; }; List<string> listURLs = new List<string>(); ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); //CSS Request foreach (string item in GetHrefsFromString(pageSource)) { if (item.Contains(".css")) { string cssSource = item.Replace(" ", "").Trim(); try { //string res = HttpHelper.GetHtmlProxy(cssSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); listURLs.Add(cssSource); } catch (Exception) { Thread.Sleep(500); try { string res = HttpHelper.GetHtmlProxy(cssSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); } catch (Exception) { } } } } //JS Request string[] scriptArr = Regex.Split(pageSource, "/script>"); foreach (string item in scriptArr) { try { if (item.Contains("static.ak.") || item.Contains("profile.ak.")) { int startIndx = item.LastIndexOf("src=") + "src=".Length + 1; int endIndx = item.IndexOf(">", startIndx) - 1; string jsSource = item.Substring(startIndx, endIndx - startIndx); //if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.")) if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.") || jsSource.StartsWith("http://profile.ak.") || jsSource.StartsWith("https://profile.ak.") || jsSource.StartsWith("http://s-profile.ak.") || jsSource.StartsWith("https://s-profile.ak.")) { try { //string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); listURLs.Add(jsSource); } catch (Exception) { Thread.Sleep(500); try { string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); } catch (Exception) { } } } } } catch (Exception) { } } string[] moreScriptArray = Regex.Split(pageSource, "\"src\":"); foreach (string item in moreScriptArray) { try { int startIndx = 1; int endIndx = item.IndexOf("\"", startIndx); string jsSource = item.Substring(startIndx, endIndx - startIndx).Replace("\\", ""); if (jsSource.Contains(".js")) { //string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); listURLs.Add(jsSource); } } catch (Exception) { } } ///IMG Request string[] imageArr = Regex.Split(pageSource, "<img"); foreach (string item in imageArr) { try { if (item.Contains("static.ak.") || item.Contains("profile.ak.")) { int startIndx = item.IndexOf("src=") + "src=".Length + 1; int endIndx = item.IndexOf("\"", startIndx + 1); string jsSource = item.Substring(startIndx, endIndx - startIndx); //if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.")) if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.") || jsSource.StartsWith("http://profile.ak.") || jsSource.StartsWith("https://profile.ak.") || jsSource.StartsWith("http://s-profile.ak.") || jsSource.StartsWith("https://s-profile.ak.")) { if (jsSource.Contains(".png") || jsSource.Contains(".gif") || jsSource.Contains(".jpg") || jsSource.Contains(".jpeg")) { try { //string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); listURLs.Add(jsSource); } catch (Exception) { Thread.Sleep(500); try { string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource)); } catch (Exception) { } } } } } } catch { }; } listURLs = listURLs.Distinct().ToList(); foreach (string item in listURLs) { try { string res = HttpHelper.GetHtmlProxy(item, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http); } catch { }; } }
public void StartAcceptInvitations(ref GlobusHttpHelper httpHelper) { try { string csrfToken = string.Empty; string userFirstName = string.Empty; string UserLastName = string.Empty; string SenderName = string.Empty; string newPagesource = string.Empty; bool isTrue = false; int startRow = 1; string pageSource = httpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/inbox/invitations/pending")); var resultForUserDetails = FindTheUserName(pageSource); try { resultForUserDetails = resultForUserDetails.Substring(resultForUserDetails.IndexOf("alt="), resultForUserDetails.IndexOf("height") - resultForUserDetails.IndexOf("alt=")).Replace("alt=", string.Empty).Replace("/", string.Empty).Trim(); userFirstName = resultForUserDetails.Split(' ')[0].Replace("\"", string.Empty); UserLastName = resultForUserDetails.Split(' ')[1].Replace("\"", string.Empty); } catch { } if (pageSource.Contains("csrfToken")) { csrfToken = pageSource.Substring(pageSource.IndexOf("csrfToken"), 100); string[] Arr = csrfToken.Split('"'); try { foreach (string item in Arr) { try { if (item.Contains("csrfToken=")) { csrfToken = item.Substring(item.IndexOf("csrfToken="), item.IndexOf("&", item.IndexOf("csrfToken=")) - item.IndexOf("csrfToken=")).Replace("csrfToken=", string.Empty).Replace("\"", string.Empty).Replace("\\", string.Empty).Trim();//Arr[2].Replace(@"\", string.Empty).Replace("//", string.Empty); break; } } catch (Exception ex) { } if (item.Contains("csrfToken=")) { csrfToken = item.Replace("csrfToken=", string.Empty).Trim(); break; } } } catch (Exception ex) { //csrfToken = Arr[0].Replace("csrfToken=", "").Replace("\\", ""); } } // For Show More //string postData1 = "pkey=inbox-invitations-pending&tcode=%5Bobject%20Arguments%5D&plist="; //string response1 = httpHelper.postFormData(new Uri("http://www.linkedin.com/lite/web-action-track?csrfToken="+csrfToken+""),postData1); //string pageSource2=httpHelper.getHtmlfromUrl(new Uri("http://www.linkedin.com/inbox/invitations/pending/more?sinceDate=1366351490125&startRow=6&count=20&showBlocked=false&ctx=inbox&rnd=1366353236172")); //*** Conver HTML to XML *******************************// #region Convert HTML to XML ChilkatHttpHelpr objhelper = new ChilkatHttpHelpr(); //xHtml contain xml data string xHtml = objhelper.ConvertHtmlToXml(pageSource); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); //xHtml. //// Iterate over all h1 tags: Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion #region Invitatin count xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "invitation-count count "); try { while ((xNode != null)) { string strvalue = xNode.AccumulateTagContent("text", "script|style"); string Invitatincount = strvalue; Log("[ " + DateTime.Now + " ] => [ Invitation Count = " + Invitatincount + " UserName = "******" ]"); Log("-----------------------------------------------------------------------------------------------------------------------------------"); break; } } catch (Exception ex) { } #endregion do { newPagesource = httpHelper.getHtmlfromUrl1(new Uri("https://www.linkedin.com/inbox/invitations?keywords=&sortBy=&startRow=" + startRow + "&subFilter=&trk=&showBlocked=false")); if (newPagesource.Contains("inbox-list")) { string inbox_list = httpHelper.GetDataWithTagValueByTagAndAttributeNameWithClass(newPagesource, "ol", "inbox-list "); if (inbox_list.Contains("<li")) { isTrue = true; string[] srrLi = Regex.Split(inbox_list, "<li"); foreach (string item in srrLi) { try { if (item.Contains("data-gid=\"")) { string data_gid = item.Substring(item.IndexOf("data-gid=\"") + 10, item.IndexOf("\"", item.IndexOf("data-gid=\"") + 10) - (item.IndexOf("data-gid=\"") + 10)).Replace("\"", string.Empty).Replace("data-gid=\"", string.Empty).Trim(); int startindex1 = item.IndexOf("alt="); string start1 = item.Substring(startindex1).Replace("alt=",string.Empty); int endindex1 = start1.IndexOf("height"); string end1 = start1.Substring(0, endindex1).Replace("\"", string.Empty).Trim(); SenderName = end1; string response2 = httpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/inbox/action?mboxItemGID=" + data_gid + "&actionType=invitationAccept&csrfToken=" + csrfToken + "&goback=%2Epiv_*1_*1_*1_*1_*1&trk=inbox-invitations-inv-accept&ctx=inbox&rnd=1366352095313")); if (response2.Contains(" are now connected")) { string SuccessMsg = string.Empty; int startindex = response2.IndexOf("<div class=\"confirmation\">"); if (startindex > 0) { try { string start = response2.Substring(startindex).Replace("<div class=\"confirmation\">", string.Empty); int endindex = start.IndexOf("<ul>"); string end = start.Substring(0, endindex); //SuccessMsg = end.Replace("<h4>", string.Empty).Replace("\"", string.Empty).Replace("</h4>", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("\"u002", "-").Replace("You", "User: "******"<h4>", string.Empty).Replace("\"", string.Empty).Replace("</h4>", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("\"u002", "-").Replace("You", "User: "******"( Name:" + userFirstName + " " + UserLastName + ") ").Trim(); } catch { } } GlobusFileHelper.AppendStringToTextfileNewLine(SuccessMsg, Globals.path_AcceptInvitationEmail); Log("[ " + DateTime.Now + " ] => [ " + SuccessMsg + " ]"); } if (!(response2.Contains(SenderName))) { Log("[ " + DateTime.Now + " ] => [ Invitation accepted from " + SenderName + " ]"); } else { //Log("There is some error !"); } } } catch (Exception ex) { } } } else { //Log("[ " + DateTime.Now + " ] => [ There is no invitation ! ]"); Log("[ " + DateTime.Now + " ] => [ No more invitations left to accept ! ]"); } } startRow = startRow + 10; } while (newPagesource.Contains("is now a connection.")); //else //{ // Log("[ " + DateTime.Now + " ] => [ There is no invitation ! ]"); //} //if (isTrue) //{ // StartAcceptInvitations(ref httpHelper); //} } catch (Exception ex) { } }
public string CreateBoard_new(string BoardName, string Category, ref PinInterestUser objPinUser) { try { string Checking = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri("https://www.pinterest.com/settings/")); string ScreenName = objPinUser.ScreenName; //ObjAccountManager.Getscreen_NameRepin(ref objPinUser); if (Checking.Contains("profileName")) { } else { ObjAccountManager.LoginPinterestAccount(ref objPinUser); } string redirectDomain = GlobusHttpHelper.valueURl.Split('.')[0]; string newHomePageUrl = redirectDomain + "." + "pinterest.com"; if (!string.IsNullOrEmpty(Checking)) { ChilkatHttpHelpr objChilkatHttpHelpr = new ChilkatHttpHelpr(); //string newpostdata = "source_url=%2F" + ScreenName + "%2F&data=%7B%22options%22%3A%7B%22name%22%3A%22" + (BoardName.Replace(" ", "+")) + "%22%2C%22category%22%3A%22other%22%2C%22description%22%3A%22%22%2C%22privacy%22%3A%22public%22%2C%22layout%22%3A%22default%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=App()%3EUserProfilePage(resource%3DUserResource(username%3D" + ScreenName + "%2C+invite_code%3Dnull))%3EUserProfileContent(resource%3DUserResource(username%3D" + objPinUser.ScreenName + "%2C+invite_code%3Dnull))%3EUserBoards()%3EGrid(resource%3DProfileBoardsResource(username%3D" + ScreenName + "))%3EGridItems(resource%3DProfileBoardsResource(username%3D" + ScreenName + "))%3EBoardCreateRep(ga_category%3Dboard_create%2C+text%3DCreate+a+board%2C+submodule%3D%5Bobject+Object%5D)%23Modal(module%3DBoardCreate())"; string newpostdata = "source_url=%2F" + ScreenName + "%2F&data=%7B%22options%22%3A%7B%22name%22%3A%22" + (BoardName.Replace(" ", "+")) + "%22%2C%22category%22%3A%22other%22%2C%22description%22%3A%22%22%2C%22privacy%22%3A%22public%22%2C%22layout%22%3A%22default%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=App%3EUserProfilePage%3EUserProfileContent%3EUserBoards%3EGrid%3EGridItems%3EBoardCreateRep(ga_category%3Dboard_create%2C+text%3DCreate+a+board%2C+submodule%3D%5Bobject+Object%5D)%23App%3EModalManager%3EModal(showCloseModal%3Dtrue%2C+mouseDownInModal%3Dfalse)"; string PostUrlBoard = redirectDomain + ".pinterest.com/resource/BoardResource/create/"; try { CreatedBoardPageSource = objPinUser.globusHttpHelper.postFormDataProxyPin(new Uri(PostUrlBoard), newpostdata, newHomePageUrl); } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } if (CreatedBoardPageSource.Contains("error\": null")) { if (newpostdata.Contains("You already have a board with that name.")) { GlobusLogHelper.log.Info(" => [ You already have a board with this name. " + BoardName + " For " + objPinUser.Username + " ]"); return(null); } try { string ModuleName = "AddBoardName"; string Status = "Board_Created"; QueryManager qm = new QueryManager(); qm.insertAccRePort(objPinUser.Username, ModuleName, "", BoardName, "", "", "", "", Status, "", "", DateTime.Now); //qm.insertBoard_AddBoardName(objPinUser.Username, ModuleName, BoardName, Status); objDelegateAccountReport(); } catch (Exception ex) { GlobusLogHelper.log.Info(" => [ Board Created " + BoardName + " ]"); return(null); } GlobusLogHelper.log.Info(" => [ Successfully Created Board " + BoardName + " For " + objPinUser.Username + " ]"); string BoardId = objAddNewPinManager.GetBoardId_Board(BoardName, ref objPinUser);//GetBoardId(BoardName, ref pinterestAccountManager); return(BoardId); } else if (CreatedBoardPageSource.Contains("You have a board with this name.")) { GlobusLogHelper.log.Info(" => [ You already have a board with this name. " + BoardName + " For " + objPinUser.Username + " ]"); string BoardId = objAddNewPinManager.GetBoardId_Board(BoardName, ref objPinUser); return(BoardId); } else { GlobusLogHelper.log.Info(" => [ Board Creation Process Failed " + BoardName + " ]"); return(CreatedBoardPageSource); } } else { GlobusLogHelper.log.Info(" => [ Login Issue " + " For " + objPinUser.Username + " ]"); } } catch (Exception ex) { GlobusLogHelper.log.Info(" => [ Board Creation Process Failed " + BoardName + " ]"); return(null); } return(null); }
public List<StructTweetIDs> NewKeywordStructDataSearchByPeople(string keyword) { try { BaseLib.GlobusRegex regx = new GlobusRegex(); int counter = 0; lst_structTweetIDs = new List<StructTweetIDs>(); string res_Get_searchURL = string.Empty; string searchURL = string.Empty; string maxid = string.Empty; string TweetId = string.Empty; string text = string.Empty; string ProfileName = string.Empty; string Location = string.Empty; string Bio = string.Empty; string website = string.Empty; string NoOfTweets = string.Empty; string Followers = string.Empty; string Followings = string.Empty; if (keyword.Trim().Contains(" ")) { keyword = keyword.Replace(" ", "+"); } startAgain: if (!RetweetFromUserName) { if (counter == 0) { //searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + ""; //searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime"; //29-4-2014 only for client it has been changed searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&mode=users"; counter++; } else { //searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; //29-4-2014 only for client it is changed searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&mode=users&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; } } else { searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + RetweetExtractcount + "&include_available_features=1&include_entities=1"; } try { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); if (string.IsNullOrEmpty(res_Get_searchURL)) { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } try { //string sjss = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor"); //splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("refresh_cursor")) { int startIndex = item.IndexOf("USER-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } if (item.Contains("scroll_cursor")) { int startIndex = item.IndexOf("USER-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } } } catch (Exception) { } } catch (Exception ex) { System.Threading.Thread.Sleep(2000); res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs); } // && !res_Get_searchURL.Contains("has_more_items\":false") if (!string.IsNullOrEmpty(res_Get_searchURL)) { //string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); //Regex.Split(res_Get_searchURL, "\"in_reply_to_status_id_str\""); string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("data-screen-name=") && !item.Contains("js-actionable-user js-profile-popup-actionable")) { //var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL); //string DataHtml = (string)avc["items_html"]; } else { //continue; } string modified_Item = "\"from_user\"" + item; string id = ""; try { int startIndex = item.IndexOf("data-user-id="); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); id = end; } catch (Exception ex) { id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } string from_user_id = ""; try { int startIndex = item.IndexOf("data-screen-name=\\\""); string start = item.Substring(startIndex).Replace("data-screen-name=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); from_user_id = end; } catch (Exception ex) { from_user_id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } string tweetUserid = string.Empty; try { int startIndex = item.IndexOf("=\\\""); string start = item.Substring(startIndex).Replace("=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end; } catch (Exception ex) { from_user_id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } StructTweetIDs structTweetIDs = new StructTweetIDs(); if (id != "null") { structTweetIDs.ID_Tweet = tweetUserid; structTweetIDs.ID_Tweet_User = id; structTweetIDs.username__Tweet_User = from_user_id; structTweetIDs.wholeTweetMessage = text; lst_structTweetIDs.Add(structTweetIDs); Log("[ " + DateTime.Now + " ] => [ " + tweetUserid + " ]"); Log("[ " + DateTime.Now + " ] => [ " + id + " ]"); Log("[ " + DateTime.Now + " ] => [ " + from_user_id + " ]"); Log("-------------------------------------------------------------------------------------------------------------------------------"); if (!File.Exists(Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv")) { GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS", Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"); } //foreach (TwitterDataScrapper.StructTweetIDs item in data) { ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr(); GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + from_user_id), "", ""); string Responce = ProfilePageSource; #region Convert HTML to XML string xHtml = objChilkat.ConvertHtmlToXml(Responce); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion int counterdata = 0; xBeginSearchAfter = null; string dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterdata == 0) { ProfileName = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else if (counterdata == 1) { website = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else { break; } // xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor"); } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim(); break; } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Location = xNode.AccumulateTagContent("text", "script|style"); break; } int counterData = 0; xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterData == 0) { // NoOfTweets = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileNav-value"); NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty); counterData++; } else if (counterData == 1) { Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty); counterData++; } else if (counterData == 2) { Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty); counterData++; } else { break; } //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor"); } if (!string.IsNullOrEmpty(from_user_id) && tweetUserid != "null") { string Id_user = tweetUserid.Replace("}]", string.Empty).Trim(); Globals.lstScrapedUserIDs.Add(Id_user); GlobusFileHelper.AppendStringToTextfileNewLine(id + "," + from_user_id + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"); GlobusFileHelper.AppendStringToTextfileNewLine(from_user_id, Globals.Path_KeywordScrapedListData + "-" + keyword + ".txt"); Log("[ " + DateTime.Now + " ] => [ " + from_user_id + "," + Id_user + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]"); } } } lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList(); if (lst_structTweetIDs.Count >= noOfRecords) { return lst_structTweetIDs; } } if (lst_structTweetIDs.Count <= noOfRecords) { maxid = lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet; if (res_Get_searchURL.Contains("has_moreitems\":false")) { return lst_structTweetIDs; } else { goto startAgain; } } else { if (res_Get_searchURL.Contains("has_more_items\":false")) { return lst_structTweetIDs; } else goto startAgain; } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return lst_structTweetIDs; }
public static string GetUserNameFromUserId(string userid) { string username = string.Empty; ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr(); clsDBQueryManager DB = new clsDBQueryManager(); DataSet ds = DB.GetUserName(userid); string user_id = string.Empty; if (ds.Tables[0].Rows.Count > 0) { foreach (DataRow dataRow in ds.Tables["tb_UsernameDetails"].Rows) { user_id = dataRow.ItemArray[0].ToString(); return user_id; } } try { string PageSource = string.Empty; if (!string.IsNullOrEmpty(userid) && NumberHelper.ValidateNumber(userid)) { PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + userid + "&include_entities=true"); if (!PageSource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.")) { try { int startIndex = PageSource.IndexOf("<screen_name>"); if (startIndex > 0) { string Start = PageSource.Substring(startIndex); int endIndex = Start.IndexOf("</screen_name>"); string End = Start.Substring(0, endIndex); username = End.Replace("<screen_name>", ""); } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } } else { username = "******"; } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return username; }
//public List<string> GetRetweetData_Scrape(string keyword, out string returnStatus) //{ // string status = string.Empty; // List<string> lst_ReTweetIDs = new List<string>(); // try // { // string searchURL = string.Empty; // if (!NumberHelper.ValidateNumber(keyword)) // { // searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + RetweetExtractcount + "&include_available_features=1&include_entities=1"; // //searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?screen_name=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true"; // //searchURL = "https://api.twitter.com/1.1/statuses/retweeted_by_user.json?screen_name=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true"; // } // else if (NumberHelper.ValidateNumber(keyword)) // { // searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?id=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true"; // } // //string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); // ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); // string res_Get_searchURL = HttpHelper.GetHtml(searchURL); // // if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && res_Get_searchURL.Contains("created_at") && !string.IsNullOrEmpty(res_Get_searchURL)) // if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && !string.IsNullOrEmpty(res_Get_searchURL)) // { // string[] arrGetRetweeted = Regex.Split(res_Get_searchURL, "js-tweet-text tweet-text"); // foreach (string item in arrGetRetweeted) // { // try // { // if (item.Contains("Retweeted by")) // { // int Startindex = item.IndexOf("\\\""); // string start = item.Substring(Startindex); // int EndIndex = start.IndexOf("\\/p"); // string End = start.Substring(0, EndIndex).Replace("\\\"", "").Replace("\\u003", "").Replace("&", ""); // End = End.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "").Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); ; // } // } // // catch { } // //} // //string[] splitRes = Regex.Split(res_Get_searchURL, "status");//Regex.Split(res_Get_searchURL, "{\"created_at\""); // //splitRes = splitRes.Skip(1).ToArray(); // //foreach (string item in splitRes) // //{ // // string Tweet = string.Empty; // // string Tweeter = string.Empty; // // try // // { // // int startIndex = item.IndexOf("<text>"); // // string start = item.Substring(startIndex); // // int endIndex = start.IndexOf("</text>"); // // string end = start.Substring(0, endIndex); // // Tweet = end.Replace("<text>", ""); // // int startOfInndex = Tweet.IndexOf(":"); // // Tweeter = Tweet.Substring(0, startOfInndex); // // Tweet = Tweet.Replace(Tweeter, ""); // // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); // } // //string data = keyword + ":" + Tweeter + ":" + Tweet.Replace(":", "^"); // //Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(data, Globals.Path_RETweetExtractor); // //lst_ReTweetIDs.Add(data); // } // } // else if (res_Get_searchURL.Contains("Rate limit exceeded")) // { // status = "Rate limit exceeded"; // } // else if (res_Get_searchURL.Contains("Sorry, that page does not exist")) // { // status = "Sorry, that page does not exist"; // } // else if (res_Get_searchURL.Contains("Not authorized")) // { // status = "Not Authorized"; // } // else if (string.IsNullOrEmpty(res_Get_searchURL)) // { // status = "Not Authorized"; // } // else // { // status = "Empty"; // } // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); // status = "Error"; // } // returnStatus = status; // return lst_ReTweetIDs; //} #endregion public List<string> GetRetweetData_Scrape(string keyword, out string returnStatus) { string status = string.Empty; List<string> lst_ReTweetIDs = new List<string>(); try { string searchURL = string.Empty; int extendendCount = 0; extendendCount = RetweetExtractcount * 10; if (!NumberHelper.ValidateNumber(keyword)) { searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + extendendCount + "&include_available_features=1&include_entities=1"; } else if (NumberHelper.ValidateNumber(keyword)) { searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?id=" + keyword + "&count=" + extendendCount + "&include_entities=true"; } //string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); string res_Get_searchURL = HttpHelper.GetHtml(searchURL); if (string.IsNullOrEmpty(res_Get_searchURL)) { res_Get_searchURL = HttpHelper.GetHtml(searchURL); } var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL); string DataHtml = string.Empty; try { DataHtml = (string)avc["items_html"]; } catch { }; if (string.IsNullOrEmpty(DataHtml)) { foreach (object data in avc) { DataHtml = data.ToString(); } } string[] splitRes = Regex.Split(DataHtml, "ProfileTweet u-textBreak js-tweet js-stream-tweet js-actionable-tweet");//Regex.Split(res_Get_searchURL, "{\"created_at\""); if (splitRes.Count() == 1) { splitRes = Regex.Split(DataHtml, "ProfileTweet-action ProfileTweet-action--more js-more-ProfileTweet-actions"); } splitRes = splitRes.Skip(1).ToArray(); GlobusRegex regx = new GlobusRegex(); foreach (string item in splitRes) { string text = string.Empty; string tweetURL = string.Empty; string tweetUserid = string.Empty; if (item.Contains("Icon--retweeted"))//retweeted Icon { try { //int startindex = item.IndexOf("ProfileTweet-text js-tweet-text u-dir"); //string start = item.Substring(startindex).Replace("ProfileTweet-text js-tweet-text u-dir", "").Replace("js-tweet-text tweet-text tweet-text-rtl\\\"", ""); //int endindex = start.IndexOf("</p>"); //if (endindex == -1) //{ // endindex = 0; // endindex = start.IndexOf("stream-item-footer"); //} //string end = start.Substring(0, endindex); //end = regx.StripTagsRegex(end); #region sonu edited code try { string[] getText = Regex.Split(item, "TweetTextSize TweetTextSize--16px js-tweet-text tweet-text"); try { text = Utils.getBetween(getText[1], ">", "<"); } catch { }; }catch{}; #endregion //text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); text = text.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "").Replace(" "," "); text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("'", "'").Replace("&", "&").Replace("=>", "=>").Replace(" ", " ").Replace("\\", "").Replace("title=", "").Replace("js-tweet-text tweet-text", "").Replace(" ", " "); string[] array = Regex.Split(text, "http"); text = string.Empty; foreach (string itemData in array) { if (!itemData.Contains("t.co")) { string data = string.Empty; if (itemData.Contains("//")) { data = ("http" + itemData).Replace(" span ", string.Empty); if (!text.Contains(itemData.Replace(" ", ""))) { text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("lang=en", string.Empty).Replace("dir=ltr data-aria-label-part=0", string.Empty); } } else { if (!text.Contains(itemData.Replace(" ", ""))) { text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("lang=en", string.Empty).Replace("dir=ltr data-aria-label-part=0", string.Empty).Replace("\n",string.Empty).Replace(" ",string.Empty); } } } } if (text.Contains("data-aria-label-part=0")) { text = globushttpHelper.getBetween(text + ":&$#@", "data-aria-label-part=0", ":&$#@").Replace(" "," "); } //data-permalink-path=" #region Scrap try { //data-permalink-path=" int startIndex = item.IndexOf("data-tweet-id=\""); string start1 = item.Substring(startIndex).Replace("data-tweet-id=\"", ""); int endIndex = start1.IndexOf("\""); string end1 = start1.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end1; } catch { }; #endregion #region Scrap Url try { //data-permalink-path=" int startIndex = item.IndexOf("data-permalink-path=\""); string start1 = item.Substring(startIndex).Replace("data-permalink-path=\"", ""); int endIndex = start1.IndexOf("\""); string end1 = start1.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetURL = "https://twitter.com" + end1; } catch { }; #endregion //Remove RT or @ mentions #region -----RT or @ mentions------- if (RemoveRTMSg && text.StartsWith("RT")) continue; if (removeAtMentions && text.Contains("@")) continue; #endregion if (lst_ReTweetIDs.Count() < RetweetExtractcount) { string txtdata = keyword + ":" + (text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("#", string.Empty)).Replace("dir=ltr", "").Replace(" ", " "); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(txtdata, Globals.Path_RETweetExtractor); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("#", string.Empty).Replace("dir=ltr", "").Trim(), Globals.Path_RETweetExtractorUpload); lst_ReTweetIDs.Add(txtdata); //generate CSV file if (!File.Exists(Globals.Path_RETweetExtractorCSV)) { string Header = "UserId" + "," + "Keyword" + "," + "Message" + "," + "URL"; GlobusFileHelper.AppendStringToTextfileNewLine(Header, Globals.Path_RETweetExtractorCSV); } string CheckReTweetExtractor = tweetUserid + "," + keyword + "," + text.Replace(","," ") + "," + tweetURL; if (!string.IsNullOrEmpty(tweetURL)) { GlobusFileHelper.AppendStringToTextfileNewLine(CheckReTweetExtractor, Globals.Path_RETweetExtractorCSV); //AddToLog_Checker("[ " + DateTime.Now + " ] => [ " + userEmail + "-" + userID + "-" + ScreanName + " ]"); } } status = "No Error"; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } } } } catch (Exception ex) { status = "Empty"; } returnStatus = status; return lst_ReTweetIDs; }
public void StartAcceptInvitations(ref GlobusHttpHelper httpHelper) { try { string csrfToken = string.Empty; string userFirstName = string.Empty; string UserLastName = string.Empty; string SenderName = string.Empty; string newPagesource = string.Empty; bool isTrue = false; int startRow = 1; string pageSource = httpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/inbox/invitations/pending")); var resultForUserDetails = FindTheUserName(pageSource); try { resultForUserDetails = resultForUserDetails.Substring(resultForUserDetails.IndexOf("alt="), resultForUserDetails.IndexOf("height") - resultForUserDetails.IndexOf("alt=")).Replace("alt=", string.Empty).Replace("/", string.Empty).Trim(); userFirstName = resultForUserDetails.Split(' ')[0].Replace("\"", string.Empty); UserLastName = resultForUserDetails.Split(' ')[1].Replace("\"", string.Empty); } catch { } if (pageSource.Contains("csrfToken")) { csrfToken = pageSource.Substring(pageSource.IndexOf("csrfToken"), 100); string[] Arr = csrfToken.Split('"'); try { foreach (string item in Arr) { try { if (item.Contains("csrfToken=")) { csrfToken = item.Substring(item.IndexOf("csrfToken="), item.IndexOf("&", item.IndexOf("csrfToken=")) - item.IndexOf("csrfToken=")).Replace("csrfToken=", string.Empty).Replace("\"", string.Empty).Replace("\\", string.Empty).Trim();//Arr[2].Replace(@"\", string.Empty).Replace("//", string.Empty); break; } } catch (Exception ex) { } if (item.Contains("csrfToken=")) { csrfToken = item.Replace("csrfToken=", string.Empty).Trim(); break; } } } catch (Exception ex) { //csrfToken = Arr[0].Replace("csrfToken=", "").Replace("\\", ""); } } // For Show More //string postData1 = "pkey=inbox-invitations-pending&tcode=%5Bobject%20Arguments%5D&plist="; //string response1 = httpHelper.postFormData(new Uri("http://www.linkedin.com/lite/web-action-track?csrfToken="+csrfToken+""),postData1); //string pageSource2=httpHelper.getHtmlfromUrl(new Uri("http://www.linkedin.com/inbox/invitations/pending/more?sinceDate=1366351490125&startRow=6&count=20&showBlocked=false&ctx=inbox&rnd=1366353236172")); //*** Conver HTML to XML *******************************// #region Convert HTML to XML ChilkatHttpHelpr objhelper = new ChilkatHttpHelpr(); //xHtml contain xml data string xHtml = objhelper.ConvertHtmlToXml(pageSource); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); //xHtml. //// Iterate over all h1 tags: Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion #region Invitatin count xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "invitation-count count "); try { while ((xNode != null)) { string strvalue = xNode.AccumulateTagContent("text", "script|style"); string Invitatincount = strvalue; Log("[ " + DateTime.Now + " ] => [ Invitation Count = " + Invitatincount + " UserName = "******" ]"); Log("-----------------------------------------------------------------------------------------------------------------------------------"); break; } } catch (Exception ex) { } #endregion do { newPagesource = httpHelper.getHtmlfromUrl1(new Uri("https://www.linkedin.com/inbox/invitations?keywords=&sortBy=&startRow=" + startRow + "&subFilter=&trk=&showBlocked=false")); if (newPagesource.Contains("inbox-list")) { string inbox_list = httpHelper.GetDataWithTagValueByTagAndAttributeNameWithClass(newPagesource, "ol", "inbox-list "); if (inbox_list.Contains("<li")) { isTrue = true; string[] srrLi = Regex.Split(inbox_list, "<li"); foreach (string item in srrLi) { try { if (item.Contains("data-gid=\"")) { string data_gid = item.Substring(item.IndexOf("data-gid=\"") + 10, item.IndexOf("\"", item.IndexOf("data-gid=\"") + 10) - (item.IndexOf("data-gid=\"") + 10)).Replace("\"", string.Empty).Replace("data-gid=\"", string.Empty).Trim(); int startindex1 = item.IndexOf("alt="); string start1 = item.Substring(startindex1).Replace("alt=", string.Empty); int endindex1 = start1.IndexOf("height"); string end1 = start1.Substring(0, endindex1).Replace("\"", string.Empty).Trim(); SenderName = end1; string response2 = httpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/inbox/action?mboxItemGID=" + data_gid + "&actionType=invitationAccept&csrfToken=" + csrfToken + "&goback=%2Epiv_*1_*1_*1_*1_*1&trk=inbox-invitations-inv-accept&ctx=inbox&rnd=1366352095313")); if (response2.Contains(" are now connected")) { string SuccessMsg = string.Empty; int startindex = response2.IndexOf("<div class=\"confirmation\">"); if (startindex > 0) { try { string start = response2.Substring(startindex).Replace("<div class=\"confirmation\">", string.Empty); int endindex = start.IndexOf("<ul>"); string end = start.Substring(0, endindex); //SuccessMsg = end.Replace("<h4>", string.Empty).Replace("\"", string.Empty).Replace("</h4>", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("\"u002", "-").Replace("You", "User: "******"<h4>", string.Empty).Replace("\"", string.Empty).Replace("</h4>", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("\"u002", "-").Replace("You", "User: "******"( Name:" + userFirstName + " " + UserLastName + ") ").Trim(); } catch { } } GlobusFileHelper.AppendStringToTextfileNewLine(SuccessMsg, Globals.path_AcceptInvitationEmail); Log("[ " + DateTime.Now + " ] => [ " + SuccessMsg + " ]"); } if (!(response2.Contains(SenderName))) { Log("[ " + DateTime.Now + " ] => [ Invitation accepted from " + SenderName + " ]"); } else { //Log("There is some error !"); } } } catch (Exception ex) { } } } else { //Log("[ " + DateTime.Now + " ] => [ There is no invitation ! ]"); Log("[ " + DateTime.Now + " ] => [ No more invitations left to accept ! ]"); } } startRow = startRow + 10; } while (newPagesource.Contains("is now a connection.")); //else //{ // Log("[ " + DateTime.Now + " ] => [ There is no invitation ! ]"); //} //if (isTrue) //{ // StartAcceptInvitations(ref httpHelper); //} } catch (Exception ex) { } }
public void scrapUserInfo(object param) { try { Array paramsArray = new object[1]; paramsArray = (Array)param; string UserName = (string)paramsArray.GetValue(0); string userId = string.Empty; string ProfileName = string.Empty; string Location = string.Empty; string Bio = string.Empty; string website = string.Empty; string NoOfTweets = string.Empty; string Followers = string.Empty; string Followings = string.Empty; string IsProfilePIc = string.Empty; ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr(); GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + UserName.Trim()), "", ""); if (string.IsNullOrEmpty(ProfilePageSource)) { ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + UserName.Trim()), "", ""); } if (string.IsNullOrEmpty(ProfilePageSource)) { AddToLog_ScrapMember("[ " + DateTime.Now + " ] => [ User " + UserName + " is not exist or page source getting null.]"); return; } if (ProfilePageSource.Contains("Account suspended")) { AddToLog_ScrapMember("[ " + DateTime.Now + " ] => [ User " + UserName + " is suspended ]"); return; } string Responce = ProfilePageSource; #region Convert HTML to XML string xHtml = objChilkat.ConvertHtmlToXml(Responce); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion int counterdata = 0; xBeginSearchAfter = null; string dataDescription = string.Empty; //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterdata == 0) { ProfileName = xNode.AccumulateTagContent("text", "script|style"); if (ProfileName.Contains("Verified account")) { ProfileName = ProfileName.Replace("Verified account", " "); } counterdata++; } else if (counterdata == 1) { website = xNode.AccumulateTagContent("text", "script|style"); if (website.Contains("Twitter Status")) { website = ""; } counterdata++; } else { break; } //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor"); } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim(); break; } xBeginSearchAfter = null; dataDescription = string.Empty; //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "location profile-field"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Location = xNode.AccumulateTagContent("text", "script|style"); break; } int counterData = 0; xBeginSearchAfter = null; dataDescription = string.Empty; //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "data-element-term", "tweet_stats"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterData == 0) { NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty); counterData++; } else if (counterData == 1) { Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty); counterData++; } else if (counterData == 2) { Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty); counterData++; } else { break; } //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor"); } try { int startindex = ProfilePageSource.IndexOf("profile_id"); string start = ProfilePageSource.Substring(startindex).Replace("profile_id", ""); int endindex = start.IndexOf(","); string end = start.Substring(0, endindex).Replace(""", "").Replace("\"", "").Replace(":", "").Trim(); userId = end.Trim(); if (userId.Length > 15) { startindex = ProfilePageSource.IndexOf("profile_id""); start = ProfilePageSource.Substring(startindex).Replace("profile_id"", ""); endindex = start.IndexOf(","); end = start.Substring(0, endindex).Replace(""", "").Replace("\"", "").Replace(":", "").Replace(";", "").Trim(); userId = end.Trim(); } } catch { } if (ProfilePageSource.Contains("default_profile_6_400x400") || ProfilePageSource.Contains("default_profile_5_400x400") || ProfilePageSource.Contains("default_profile_4_400x400") || ProfilePageSource.Contains("default_profile_3_400x400") || ProfilePageSource.Contains("default_profile_2_400x400") || ProfilePageSource.Contains("default_profile_1_400x400") || ProfilePageSource.Contains("default_profile_0_400x400")) { IsProfilePIc = "No"; } else { IsProfilePIc = "Yes"; } if (!File.Exists(Globals.Path_UserListInfoData)) { GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS, ProfilePic", Globals.Path_UserListInfoData); } if (!string.IsNullOrEmpty(UserName)) { //string Id_user = item.ID_Tweet_User.Replace("}]", string.Empty).Trim(); //Globals.lstScrapedUserIDs.Add(Id_user); GlobusFileHelper.AppendStringToTextfileNewLine(userId + "," + UserName + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", "") + "," + IsProfilePIc, Globals.Path_UserListInfoData); AddToLog_ScrapMember("[ " + DateTime.Now + " ] => [ " + userId + "," + UserName + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ," + IsProfilePIc + "]"); } } catch { } }