public void chekLicense()
{
#region FreeTrial
try
{
ChilkatHttpHelpr objCjilkatHttpHelper = new ChilkatHttpHelpr();
string strdateTime_DataBase = ("2016-02-06 23:59:59").ToString();// string strdateTime_DataBase = ("2016-02-04 23:59:59").ToString();
DateTime dt = DateTime.Parse(strdateTime_DataBase);
strdateTime_DataBase = dt.ToString("yyyy-MM-dd hh:mm:ss");
string dateTime = objCjilkatHttpHelper.GetHtml("http://licensing.facedominator.com/licensing/FD/Datetime.php");
DateTime dt_now = DateTime.Parse(dateTime);
TimeSpan dt_Difference = dt_now.Subtract(dt);
if (dt_Difference.Days >= 1)
{
//ModernDialog.ShowMessage("Your Trial Version of Software Has Been Expired!!","Warning Message",MessageBoxButton.OK,this.OwnedWindows);
MessageBox.Show("Your Trial Version of Software Has Been Expired!!");
mainFormReference.Close();
return;
}
}
catch { }
#endregion
}
/// <summary>
/// This Method is used for check given proxy has working or not And Return those Proxy set for use in their method
/// </summary>
/// <param name="IPaddress">Given IpAddress </param>
/// <param name="Port">Given Port</param>
/// <param name="Username">Given Username</param>
/// <param name="Password">Given Password</param>
/// <param name="CheckingUrl">Given Url where we check the Proxy</param>
/// <returns></returns>
public Proxies.Proxy GetSingleUserProxy(string IPaddress, string Port, string Username, string Password, string CheckingUrl)
{
ChilkatHttpHelpr objChilkatHelper = new ChilkatHttpHelpr();
Proxy objProxy = new Proxy();
try
{
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
string respnceTime = string.Empty;
timer.Start();
string Responce = objChilkatHelper.GetHtmlProxy(CheckingUrl, IPaddress, Port, Username, Password);
timer.Stop();
TimeSpan timeTaken = timer.Elapsed;
respnceTime = timeTaken.ToString();
if (!string.IsNullOrEmpty(Responce) && objChilkatHelper.http.LastStatus != 404 && objChilkatHelper.http.LastStatus != 500 && !Responce.Contains("Internet Explorer cannot display the webpage"))
{
objProxy.IpAddress = IPaddress;
objProxy.Port = Port;
objProxy.UserName = Username;
objProxy.Password = Password;
}
else
{
}
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error : " + ex.StackTrace);
}
return(objProxy);
}
/// <summary>
/// This Method is used for check given proxy has working or not ?
/// </summary>
/// <param name="IPaddress">Given IpAddress </param>
/// <param name="Port">Given Port</param>
/// <param name="Username">Given Username</param>
/// <param name="Password">Given Password</param>
/// <param name="CheckingUrl">Given Url where we check the Proxy</param>
/// <returns></returns>
public bool CheckProxy(string IPaddress, string Port, string Username, string Password, string CheckingUrl)
{
ChilkatHttpHelpr objChilkatHelper = new ChilkatHttpHelpr();
bool status = false;
try
{
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
string ResponseTime = string.Empty;
timer.Start();
string Response = objChilkatHelper.GetHtmlProxy(CheckingUrl, IPaddress, Port, Username, Password);
timer.Stop();
TimeSpan timeTaken = timer.Elapsed;
ResponseTime = timeTaken.ToString();
if (!string.IsNullOrEmpty(Response) && objChilkatHelper.http.LastStatus != 404 && objChilkatHelper.http.LastStatus != 500 && !Response.Contains("Internet Explorer cannot display the webpage"))
{
status = true;
}
else
{
status = false;
}
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error : " + ex.StackTrace);
}
return(status);
}
public string RegisterUser(string Username, string Password, string cpuID, string TransactionID, string Email, string servr)
{
ChilkatHttpHelpr HttpHelpr = new ChilkatHttpHelpr();
string res = string.Empty;
try
{
string regUrl = "http://" + servr + "/register.php?user="******"&pass="******"&cpid=" + cpuID + "&transid=" + TransactionID + "&email=" + Email + "&LicType=" + Globals.licType + " ";
res = HttpHelpr.GetHtml("http://" + servr + "/register.php?user="******"&pass="******"&cpid=" + cpuID + "&transid=" + TransactionID + "&email=" + Email + "&LicType=" + Globals.licType + "");
if (string.IsNullOrEmpty(res))
{
System.Threading.Thread.Sleep(1000);
res = HttpHelpr.GetHtml("http://" + servr + "/register.php?user="******"&pass="******"&cpid=" + cpuID + "&transid=" + TransactionID + "&email=" + Email + "&LicType=" + Globals.licType + " ");
}
if (string.IsNullOrEmpty(res))
{
MessageBox.Show("Error Connecting to Facedominator Server,Please check if www.facedominator.com is opening for you.");
// Application.Exit();
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
return(res);
}
public static string GetUserIDFromUsername(string username, out string Status)
{
string GetStatus = string.Empty;
Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper();
clsDBQueryManager DB = new clsDBQueryManager();
DataSet ds = DB.GetUserId(username);
string user_id = string.Empty;
if (ds.Tables[0].Rows.Count > 0)
{
foreach (DataRow dataRow in ds.Tables["tb_UsernameDetails"].Rows)
{
user_id = dataRow.ItemArray[0].ToString();
Status = "No Error";
return(user_id);
}
}
try
{
string id = string.Empty;
ChilkatHttpHelpr httpHelper1 = new ChilkatHttpHelpr();
string pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username);
if (!pagesource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.") && !pagesource.Contains("Sorry, that page does not exist") && !pagesource.Contains("User has been suspended"))
{
int length = pagesource.IndexOf("</id>");
id = pagesource.Substring(pagesource.IndexOf("<id>"), length - pagesource.IndexOf("<id>")).Replace("<id>", "");
user_id = id;
GetStatus = "No Error";
}
else if (pagesource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour."))
{
GetStatus = "Rate limit exceeded";
}
else if (pagesource.Contains("Sorry, that page does not exist"))
{
GetStatus = "Sorry, that page does not exist";
}
else if (pagesource.Contains("User has been suspended"))
{
GetStatus = "User has been suspended";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserIDFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserIDFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
GetStatus = "Error";
}
Status = GetStatus;
return(user_id);
}
public List <string> GetFollowers(string userID, out string ReturnStatus)
{
try
{
string FollowingUrl = string.Empty;
if (NumberHelper.ValidateNumber(userID))
{
FollowingUrl = "https://api.twitter.com/1/followers/ids.json?cursor=-1&id=" + userID + "";//"https://api.twitter.com/1/friends/ids.json?cursor=-1&screen_name=SocioPro";
}
else
{
FollowingUrl = "https://api.twitter.com/1/followers/ids.json?cursor=-1&screen_name=" + userID + "";//"https://api.twitter.com/1/friends/ids.json?cursor=-1&screen_name=SocioPro";
}
#region gs http helper code
//https://api.twitter.com/1/following/ids.json?cursor=-1&screen_name=SocioPro
//string Data = globushttpHelper.getHtmlfromUrl(new Uri(FollowingUrl), "", "");
#endregion
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
string Data = HttpHelper.GetHtml(FollowingUrl);
if (!Data.Contains("Rate limit exceeded") && !Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}"))
{
int FirstPoint = Data.IndexOf("[");
int SecondPoint = Data.IndexOf("]");
string FollowingIds = Data.Substring(FirstPoint, SecondPoint - FirstPoint).Replace("[", string.Empty).Replace("]", string.Empty);
List <string> lstIds = FollowingIds.Split(',').ToList();
ReturnStatus = "No Error";
return(lstIds);
}
else if (Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}"))
{
ReturnStatus = "Sorry, that page does not exist :" + userID;
return(new List <string>());
}
else
{
ReturnStatus = "Error";
return(new List <string>());
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowers() -- " + userID + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowers() -- " + userID + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
ReturnStatus = "Error";
return(new List <string>());
}
}
/// <summary> Get User details for While/blacklist User
/// Get User details for While/blacklist User
/// </summary>
/// <param name="username">User Screan Name OR User ID</param>
/// <returns>Details of Users</returns>
#region Get User details for While/blacklist User
public static Dictionary <string, string> GetUserDetails(string username)
{
ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
Dictionary <string, string> dataLst = new Dictionary <string, string>();
try
{
string PageSource = null;
if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username))
{
PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?user_id=" + username + "&include_entities=true");
}
else
{
PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?screen_name=" + username + "&include_entities=true");
}
JObject jobj = JObject.Parse(PageSource);
string id = jobj["id"].ToString().Replace("\"", string.Empty);
dataLst.Add("id", id);
string name = jobj["name"].ToString().Replace("\"", string.Empty);
dataLst.Add("name", name);
string screen_name = jobj["screen_name"].ToString().Replace("\"", string.Empty);
dataLst.Add("screen_name", screen_name);
string location = jobj["location"].ToString().Replace("\"", string.Empty);
dataLst.Add("location", location);
string description = jobj["description"].ToString().Replace("\"", string.Empty);
dataLst.Add("description", description);
string followers_count = jobj["followers_count"].ToString().Replace("\"", string.Empty);
dataLst.Add("followers_count", followers_count);
string friends_count = jobj["friends_count"].ToString().Replace("\"", string.Empty);
dataLst.Add("friends_count", friends_count);
string statuses_count = jobj["statuses_count"].ToString().Replace("\"", string.Empty);
dataLst.Add("statuses_count", statuses_count);
string created_at = jobj["created_at"].ToString().Replace("\"", string.Empty);
dataLst.Add("created_at", created_at);
string time_zone = jobj["time_zone"].ToString().Replace("\"", string.Empty);
dataLst.Add("time_zone", time_zone);
string profile_image_url = jobj["profile_image_url"].ToString().Replace("\"", string.Empty);
dataLst.Add("profile_image_url", profile_image_url);
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return(dataLst);
}
public List <string> GetHashTags(out string returnStatus)
{
List <string> HashTags = new List <string>();
try
{
//string pagesource = globushttpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/trends/daily.json"), "", "");
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
string pagesource = HttpHelper.GetHtml("https://api.twitter.com/1/trends/daily.json");
if (!pagesource.Contains("Rate limit exceeded"))
{
string[] array = Regex.Split(pagesource, "\"name\":");
array = array.Skip(1).ToArray();
foreach (string item in array)
{
try
{
int startindex = item.IndexOf("\"");
string Start = item.Substring(startindex);
int endIndex = Start.IndexOf("\",");
string End = Start.Substring(0, endIndex).Replace("\"", "");
string HashTag = End;
if (HashTag.Contains("#"))
{
HashTags.Add(HashTag);
}
}
catch (Exception ex)
{
returnStatus = "Error";
return(new List <string>());
}
}
returnStatus = "No Error";
return(HashTags);
}
else
{
returnStatus = "Error";
return(new List <string>());
}
}
catch (Exception ex)
{
returnStatus = "Error";
return(new List <string>());
}
}
public string GetUserLastTweetDate(string userid)
{
string date = "";
try
{
string user_id = string.Empty;
string searchURL = string.Empty;
if (!NumberHelper.ValidateNumber(userid))
{
searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + user_id + "&count=" + TweetExtractCount;
}
else if (NumberHelper.ValidateNumber(userid))
{
searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&screen_name=" + user_id + "&count=" + TweetExtractCount;
}
//string searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + user_id + "&count=" + TweetExtractCount;
ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
//string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
string res_Get_searchURL = httpHelper.GetHtml(searchURL);
string[] splitRes = Regex.Split(res_Get_searchURL, "{\"created_at");//Regex.Split(res_Get_searchURL, "{\"created_at\"");
splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
//string text = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "<text>");
string modified_Item = "{\"created_at" + item;
date = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "created_at");//Globussoft.GlobusHttpHelper.parseText(item);
if (date.Contains("+"))
{
date = date.Remove(date.IndexOf("+")).Trim();
}
if (!string.IsNullOrEmpty(date))
{
return(date);
}
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserLastTweetDate() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserLastTweetDate() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return(date);
}
public static string GetUserNameFromUserId(string userid)
{
string username = string.Empty;
ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
try
{
string PageSource = string.Empty;
if (!string.IsNullOrEmpty(userid) && NumberHelper.ValidateNumber(userid))
{
PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + userid + "&include_entities=true");
if (!PageSource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour."))
{
try
{
int startIndex = PageSource.IndexOf("<screen_name>");
if (startIndex > 0)
{
string Start = PageSource.Substring(startIndex);
int endIndex = Start.IndexOf("</screen_name>");
string End = Start.Substring(0, endIndex);
username = End.Replace("<screen_name>", "");
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
else
{
username = "******";
}
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return(username);
}
public string GetUserLastTweetDate(string userid)
{
string date = "";
try
{
string user_id = string.Empty;
string searchURL = string.Empty;
if (!NumberHelper.ValidateNumber(userid))
{
searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + user_id + "&count=" + TweetExtractCount;
}
else if (NumberHelper.ValidateNumber(userid))
{
searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&screen_name=" + user_id + "&count=" + TweetExtractCount;
}
//string searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + user_id + "&count=" + TweetExtractCount;
ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
//string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
string res_Get_searchURL = httpHelper.GetHtml(searchURL);
string[] splitRes = Regex.Split(res_Get_searchURL, "{\"created_at");//Regex.Split(res_Get_searchURL, "{\"created_at\"");
splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
//string text = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "<text>");
string modified_Item = "{\"created_at" + item;
date = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "created_at");//Globussoft.GlobusHttpHelper.parseText(item);
if (date.Contains("+"))
{
date = date.Remove(date.IndexOf("+")).Trim();
}
if (!string.IsNullOrEmpty(date))
{
return date;
}
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserLastTweetDate() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserLastTweetDate() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return date;
}
public string Login()
{
Log("[ " + DateTime.Now + " ] => [ Logging in with Account : " + Username + " ]");
string Status = "Failed";
try
{
string firstUrl = "https://api.instagram.com/oauth/authorize/?client_id=9d836570317f4c18bca0db6d2ac38e29&redirect_uri=http://websta.me/&response_type=code&scope=comments+relationships+likes";
//https://instagram.com/oauth/authorize/?client_id=9d836570317f4c18bca0db6d2ac38e29&redirect_uri=http://websta.me/&response_type=code&scope=comments+relationships+likes
string secondURL = "https://instagram.com/oauth/authorize/?client_id=9d836570317f4c18bca0db6d2ac38e29&redirect_uri=http://websta.me/&response_type=code&scope=comments+relationships+likes";
string res_secondURL = string.Empty;
ChilkatHttpHelpr objchilkat = new ChilkatHttpHelpr();
if (!string.IsNullOrEmpty(proxyAddress) && !string.IsNullOrEmpty(proxyPort))
{
try
{
//res_secondURL = httpHelper.getHtmlfromUrlProxy(new Uri(secondURL), proxyAddress, int.Parse(proxyPort), proxyUsername, proxyPassword);
res_secondURL = objchilkat.GetHtmlProxy(secondURL, proxyAddress, proxyPort, proxyUsername, proxyPassword);
}
catch { };
}
else
{
res_secondURL = httpHelper.getHtmlfromUrl(new Uri(secondURL), "");
}
//string authlogin = "******";
//string res_nextUrl1 = httpHelper.getHtmlfromUrl(new Uri(authlogin), "");
string nextUrl = string.Empty;
string res_nextUrl = string.Empty;
if (!string.IsNullOrEmpty(res_secondURL))
{
nextUrl = "https://instagram.com/accounts/login/?force_classic_login=&next=/oauth/authorize/%3Fclient_id%3D9d836570317f4c18bca0db6d2ac38e29%26redirect_uri%3Dhttp%3A//websta.me/%26response_type%3Dcode%26scope%3Dcomments%2Brelationships%2Blikes";
res_nextUrl = httpHelper.getHtmlfromUrl(new Uri(nextUrl), "");//postFormDataProxy
}
else
{
Log("[ " + DateTime.Now + " ] => [ Logged in Failed with Account :" + Username + " ]");
Status = "Failed";
this.LoggedIn = false;
}
//string nextUrl = "https://instagram.com/accounts/login/?force_classic_login=&next=/oauth/authorize/%3Fclient_id%3D9d836570317f4c18bca0db6d2ac38e29%26redirect_uri%3Dhttp%3A//websta.me/%26response_type%3Dcode%26scope%3Dcomments%2Brelationships%2Blikes";
//string res_nextUrl = httpHelper.getHtmlfromUrl(new Uri(nextUrl), "");//postFormDataProxy
//int FirstPointToken_nextUrl1 = res_nextUrl.IndexOf("authorize");
//string FirstTokenSubString_nextUrl1 = res_nextUrl.Substring(FirstPointToken_nextUrl1);
//int SecondPointToken_nextUrl1 = FirstTokenSubString_nextUrl1.IndexOf("%");
//this.Token = FirstTokenSubString_nextUrl1.Substring(0, SecondPointToken_nextUrl1).Replace("Authorize", string.Empty).Replace("value=", string.Empty).Replace("\"", string.Empty).Replace("'", string.Empty).Trim();
try
{
int FirstPointToken_nextUrl = res_nextUrl.IndexOf("csrfmiddlewaretoken");//csrfmiddlewaretoken
string FirstTokenSubString_nextUrl = res_nextUrl.Substring(FirstPointToken_nextUrl);
int SecondPointToken_nextUrl = FirstTokenSubString_nextUrl.IndexOf("/>");
this.Token = FirstTokenSubString_nextUrl.Substring(0, SecondPointToken_nextUrl).Replace("csrfmiddlewaretoken", string.Empty).Replace("value=", string.Empty).Replace("\"", string.Empty).Replace("'", string.Empty).Trim();
}
catch { };
string login = "******";
//string authlogin= "******";
//string postauthorize = "csrfmiddlewaretoken=" + this.Token + "&allow=Authorize";
//http://websta.me/?code=90786edc0a4844b79a3d9289d27360d5
string postdata_Login = "******" + this.Token + "&username="******"&password="******"";
string res_postdata_Login = httpHelper.postFormData(new Uri(login), postdata_Login, login, "");
string autho = "https://instagram.com/oauth/authorize/?scope=comments+likes+relationships&redirect_uri=http%3A%2F%2Fwww.gramfeed.com%2Foauth%2Fcallback%3Fpage%3D&response_type=code&client_id=b59fbe4563944b6c88cced13495c0f49";
if (res_postdata_Login.Contains("Please enter a correct username and password"))
{
Status = "Failed";
this.LoggedIn = false;
}
else if (res_postdata_Login.Contains("requesting access to your Instagram account") || postdata_Login.Contains("is requesting to do the following"))
{
Status = "AccessIssue";
}
else if (res_postdata_Login.Contains("logout") || postdata_Login.Contains("LOG OUT"))
{
Log("[ " + DateTime.Now + " ] => [ Logged in with Account :" + Username + " ]");
Status = "Success";
this.LoggedIn = true;
}
//nameval.Clear();
return(Status);
}
catch (Exception ex)
{
return(ex.Message);
}
}
private void btnStart_Searching_Click(object sender, EventArgs e)
{
AllOfTheseWords = (txtAllofTheseKeywords.Text).ToString();
ThisExtractPhrase = (txtThisExactPhrase.Text).ToString();
AnyOfTheseWords = (txtAnyOfTheseWords.Text).ToString();
TheseHashTags = (txtTheseHashTags.Text).ToString();
NoneOfTheseWords = (txtNoneofTheseWords.Text).ToString();
FromTheseAccounts = (txtFromTheseAccounts.Text).ToString();
ToTheseAccounts = (txtToTheseAccounts.Text).ToString();
MentionTheseAccounts = (txtMentioningTheseAccounts.Text).ToString();
NearThisPlace = (txtNearThisPlace.Text).ToString();
AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => Process Started");
try
{
if (string.IsNullOrEmpty(ThisExtractPhrase))
{
ThisExtractPhrase = "";
}
else
{
ThisExtractPhrase = "%20%22" + ThisExtractPhrase;
}
}
catch { }
try
{
if (string.IsNullOrEmpty(AnyOfTheseWords))
{
AnyOfTheseWords = "";
}
else
{
AnyOfTheseWords = "%22%20" + AnyOfTheseWords;
}
}
catch
{ }
try
{
if (string.IsNullOrEmpty(TheseHashTags))
{
TheseHashTags = "";
}
else
{
TheseHashTags = "%20%23" + TheseHashTags;
}
}
catch
{ }
try
{
if (string.IsNullOrEmpty(NoneOfTheseWords))
{
NoneOfTheseWords = "";
}
else
{
NoneOfTheseWords = "%20-" + NoneOfTheseWords;
}
}
catch
{ }
try
{
if (string.IsNullOrEmpty(FromTheseAccounts))
{
FromTheseAccounts = "";
}
else
{
FromTheseAccounts = "%20from%3A" + FromTheseAccounts;
}
}
catch
{ }
try
{
if (string.IsNullOrEmpty(ToTheseAccounts))
{
ToTheseAccounts = "";
}
else
{
ToTheseAccounts = "%20to%3A" + ToTheseAccounts;
}
}
catch
{ }
try
{
if (string.IsNullOrEmpty(MentionTheseAccounts))
{
MentionTheseAccounts = "";
}
else
{
MentionTheseAccounts = "%20%40" + MentionTheseAccounts;
}
}
catch
{ }
try
{
if (string.IsNullOrEmpty(NearThisPlace))
{
NearThisPlace = "";
}
else
{
NearThisPlace = "%20near%3A%22" + NearThisPlace;
}
}
catch
{ }
try
{
if (!string.IsNullOrEmpty(txtAllofTheseKeywords.Text))
{
#region Commented
//try
//{
// string Url = "https://twitter.com/search?f=realtime&q=" + AllOfTheseWords + ThisExtractPhrase + AnyOfTheseWords + NoneOfTheseWords + TheseHashTags + _selectedLanguage + FromTheseAccounts + ToTheseAccounts + MentionTheseAccounts + NearThisPlace + "%22%20within%3A15mi&src=typd";
// string response = _GlobusHttpHelper.getHtmlfromUrl(new Uri(Url), "", "");
//}
//catch { } public List<StructTweetIDs> NewKeywordStructDataForSearchByKeyword(string keyword)
#endregion
{
try
{
BaseLib.GlobusRegex regx = new GlobusRegex();
int counter = 0;
string res_Get_searchURL = string.Empty;
string searchURL = string.Empty;
string maxid = string.Empty;
string TweetId = string.Empty;
string text = string.Empty;
string ProfileName = string.Empty;
string Location = string.Empty;
string Bio = string.Empty;
string website = string.Empty;
string NoOfTweets = string.Empty;
string Followers = string.Empty;
string Followings = string.Empty;
int noOfRecords = 0;
try
{
noOfRecords = int.Parse(txtNoOfRecords.Text);
}
catch { }
startAgain:
if (counter == 0)
{
searchURL = "https://twitter.com/i/search/timeline?q=" + AllOfTheseWords + ThisExtractPhrase + AnyOfTheseWords + NoneOfTheseWords + TheseHashTags + _selectedLanguage + FromTheseAccounts + ToTheseAccounts + MentionTheseAccounts + NearThisPlace + "%22%20within%3A15mi&src=typd" + "&f=realtime";
counter++;
}
else
{
searchURL = "https://twitter.com/i/search/timeline?q=" + AllOfTheseWords + ThisExtractPhrase + AnyOfTheseWords + NoneOfTheseWords + TheseHashTags + _selectedLanguage + FromTheseAccounts + ToTheseAccounts + MentionTheseAccounts + NearThisPlace + "%22%20within%3A15mi&src=typd" + "&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + "";
}
try
{
res_Get_searchURL = _GlobusHttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => Finding results for entered details ");
if (string.IsNullOrEmpty(res_Get_searchURL))
{
res_Get_searchURL = _GlobusHttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
}
try
{
//string sjss = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor");
//splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
if (item.Contains("refresh_cursor"))
{
int startIndex = item.IndexOf("TWEET-");
string start = item.Substring(startIndex).Replace("data-user-id=\\\"", "");
int endIndex = start.IndexOf("\"");
string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "");
TweetId = end;
}
if (item.Contains("scroll_cursor"))
{
int startIndex = item.IndexOf("TWEET-");
string start = item.Substring(startIndex).Replace("data-user-id=\\\"", "");
int endIndex = start.IndexOf("\"");
string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "");
TweetId = end;
}
}
}
catch (Exception)
{
}
}
catch (Exception ex)
{
System.Threading.Thread.Sleep(2000);
res_Get_searchURL = _GlobusHttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
}
// && !res_Get_searchURL.Contains("has_more_items\":false")
if (!string.IsNullOrEmpty(res_Get_searchURL))
{
//string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); //Regex.Split(res_Get_searchURL, "\"in_reply_to_status_id_str\"");
string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id");
splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
if (item.Contains("data-screen-name=") && !item.Contains("js-actionable-user js-profile-popup-actionable"))
{
//var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL);
//string DataHtml = (string)avc["items_html"];
}
else
{
continue;
}
string modified_Item = "\"from_user\"" + item;
string id = "";
try
{
int startIndex = item.IndexOf("data-user-id=");
string start = item.Substring(startIndex).Replace("data-user-id=\\\"", "");
int endIndex = start.IndexOf("\\\"");
string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "");
id = end;
//lst_structTweetIDs.Add(id);
AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => User Id " + id);
}
catch (Exception ex)
{
id = "null";
//Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
}
string from_user_id = "";
try
{
int startIndex = item.IndexOf("data-screen-name=\\\"");
string start = item.Substring(startIndex).Replace("data-screen-name=\\\"", "");
int endIndex = start.IndexOf("\\\"");
string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", "");
from_user_id = end;
AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => User ScreenName " + from_user_id);
}
catch (Exception ex)
{
from_user_id = "null";
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper);
}
string tweetUserid = string.Empty;
try
{
int startIndex = item.IndexOf("=\\\"");
string start = item.Substring(startIndex).Replace("=\\\"", "");
int endIndex = start.IndexOf("\\\"");
string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", "");
tweetUserid = end;
AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => Tweet Id " + tweetUserid);
}
catch (Exception ex)
{
from_user_id = "null";
}
///Tweet Text
#region Commented
//try
//{
// int startindex = item.IndexOf("js-tweet-text tweet-text\"");
// if (startindex == -1)
// {
// startindex = 0;
// startindex = item.IndexOf("js-tweet-text tweet-text");
// }
// string start = item.Substring(startindex).Replace("js-tweet-text tweet-text\"", "").Replace("js-tweet-text tweet-text tweet-text-rtl\"", "");
// int endindex = start.IndexOf("</p>");
// if (endindex == -1)
// {
// endindex = 0;
// endindex = start.IndexOf("stream-item-footer");
// }
// string end = start.Substring(0, endindex);
// end = regx.StripTagsRegex(end);
// text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "");
// text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", "");
// string[] array = Regex.Split(text, "http");
// text = string.Empty;
// foreach (string itemData in array)
// {
// if (!itemData.Contains("t.co"))
// {
// string data = string.Empty;
// if (itemData.Contains("//"))
// {
// data = ("http" + itemData).Replace(" span ", string.Empty);
// if (!text.Contains(itemData.Replace(" ", "")))// && !data.Contains("class") && !text.Contains(data))
// {
// text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty);
// }
// }
// else
// {
// if (!text.Contains(itemData.Replace(" ", "")))
// {
// text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("js-tweet-text tweet-text", "");
// }
// }
// }
// }
//}
//catch { };
#endregion
twtboardpro.TwitterDataScrapper.StructTweetIDs structTweetIDs = new twtboardpro.TwitterDataScrapper.StructTweetIDs();
if (id != "null")
{
structTweetIDs.ID_Tweet = tweetUserid;
structTweetIDs.ID_Tweet_User = id;
structTweetIDs.username__Tweet_User = from_user_id;
structTweetIDs.wholeTweetMessage = text;
lst_structTweetIDs.Add(structTweetIDs);
}
//if (!File.Exists(Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"))
//{
// GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS", Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv");
//}
{
ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr();
GlobusHttpHelper HttpHelper = new GlobusHttpHelper();
string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + from_user_id), "", "");
string Responce = ProfilePageSource;
#region Convert HTML to XML
string xHtml = objChilkat.ConvertHtmlToXml(Responce);
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
int counterdata = 0;
xBeginSearchAfter = null;
string dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterdata == 0)
{
ProfileName = xNode.AccumulateTagContent("text", "script|style");
counterdata++;
}
else if (counterdata == 1)
{
website = xNode.AccumulateTagContent("text", "script|style");
counterdata++;
}
else
{
break;
}
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor");
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim();
break;
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Location = xNode.AccumulateTagContent("text", "script|style");
break;
}
int counterData = 0;
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterData == 0)
{
// NoOfTweets = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileNav-value");
NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty);
counterData++;
}
else if (counterData == 1)
{
Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty);
counterData++;
}
else if (counterData == 2)
{
Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty);
counterData++;
}
else
{
break;
}
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
}
if (!string.IsNullOrEmpty(from_user_id) && tweetUserid != "null")
{
string Id_user = tweetUserid.Replace("}]", string.Empty).Trim();
Globals.lstScrapedUserIDs.Add(Id_user);
// GlobusFileHelper.AppendStringToTextfileNewLine(id + "," + from_user_id + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv");
// Log("[ " + DateTime.Now + " ] => [ " + from_user_id + "," + Id_user + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]");
}
}
lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList();
if (lst_structTweetIDs.Count >= noOfRecords)
{
// return lst_structTweetIDs;
}
}
if (lst_structTweetIDs.Count <= noOfRecords)
{
maxid = lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet;
if (res_Get_searchURL.Contains("has_moreitems\":false"))
{
}
else
{
goto startAgain;
}
}
else
{
if (res_Get_searchURL.Contains("has_more_items\":false"))
{
}
else
goto startAgain;
}
}
}
catch (Exception ex)
{
}
}
}
}
catch
{ }
}
public static string GetPhotoFromUsername(string username)
{
//Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper();
ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
string user_id = string.Empty;
string containsImage = "false";
try
{
string ImageLink = string.Empty;
string PageSource = string.Empty;
//http://a0.twimg.com/sticky/default_profile_images/default_profile_0_normal.png
if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username))
{
//PageSource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?user_id=" + username + "&include_entities=true"), "", "");
PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + username + "&include_entities=true");
}
else
{
//PageSource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?screen_name=" + username + "&include_entities=true"), "", "");
PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username + "&include_entities=true");
}
if (!string.IsNullOrEmpty(PageSource))
{
try
{
int indexStart = PageSource.IndexOf("<profile_image_url>");
if (PageSource.Contains("<profile_image_url>"))
{
string start = PageSource.Substring(indexStart);
int endIndex = start.IndexOf("</profile_image_url>");
string end = start.Substring(0, endIndex);
ImageLink = end.Replace("<profile_image_url>", "");
}
if (!string.IsNullOrEmpty(ImageLink) && ImageLink.Contains("/sticky/default_profile_images/default_profile"))
{
containsImage = "false";
}
else if (PageSource.Contains("Sorry, that page does not exist"))
{
containsImage = "false";
}
else
{
containsImage = "true";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
else if (PageSource.Contains("Clients may not make more than 150 requests per hour") && PageSource.Contains("Rate limit exceeded"))
{
containsImage = "Rate limit exceeded";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return(containsImage);
}
public void getpageSourceFromProxy(object item)
{
if (proxyStop)
{
return;
}
try
{
Thread.CurrentThread.IsBackground = true;
lstProxyThread.Add(Thread.CurrentThread);
lstProxyThread = lstProxyThread.Distinct().ToList();
}
catch { }
countParseProxiesThreads++;
Array Item_value = (Array)item;
string ClGlobul_ProxyList_item = (string)Item_value.GetValue(0);
DemoStagramPro.frm_stagram frm1 = new DemoStagramPro.frm_stagram();
Globussoft.GlobDramProHttpHelper GlobusHttpHelper = new Globussoft.GlobDramProHttpHelper();
ChilkatHttpHelpr objchilkat = new ChilkatHttpHelpr();
string proxyad = string.Empty;
string proxyport = string.Empty;
string proxyusername = string.Empty;
string proxyPassword = string.Empty;
string pagesource1 = string.Empty;
string pagesource = string.Empty;
try
{
string[] proxyLst = ClGlobul_ProxyList_item.Split(':');
if (proxyLst.Count() > 3)
{
proxyad = proxyLst[0];
proxyport = proxyLst[1];
proxyusername = proxyLst[2];
proxyPassword = proxyLst[3];
}
else if (proxyLst.Count() > 0 && proxyLst.Count() < 3)
{
proxyad = proxyLst[0];
proxyport = proxyLst[1];
}
else
{
return;
}
try
{
if (proxyStop)
{
return;
}
//pagesource1 = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri("http://websta.me/login"), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword);
pagesource1 = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri("http://web.stagram.com/"), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword);
}
catch { };
try
{
// pagesource1 = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri("http://web.stagram.com/"), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword);
pagesource1 = objchilkat.GetHtmlProxy("http://web.stagram.com/", proxyad, proxyport, proxyusername, proxyPassword);
}
catch { };
//int FirstPointClientId = pagesource1.IndexOf("client_id=");
//string FirstClientIdSubString = pagesource1.Substring(FirstPointClientId);
//int SecondPointClientId = FirstClientIdSubString.IndexOf("&redirect_uri=");
//string ClientId = FirstClientIdSubString.Substring(0, SecondPointClientId).Replace("'", string.Empty).Replace("client_id=", string.Empty).Trim();
//string LoginUrl = "https://instagram.com/accounts/login/?next=/oauth/authorize/%3Fclient_id%3D" + ClientId + "%26redirect_uri%3Dhttp%253A%252F%252Fweb.stagram.com%252F%26response_type%3Dcode%26scope%3Dlikes%2Bcomments%2Brelationships";
//pagesource = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri(LoginUrl), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword);
//if (string.IsNullOrEmpty(pagesource))
//{
// pagesource = string.Empty;
// pagesource = GlobusHttpHelper.getHtmlfromUrlProxy(new Uri(LoginUrl), proxyad, Convert.ToInt16(proxyport), proxyusername, proxyPassword);
//}
//ADD in List list of Finally chacked.....
if (!string.IsNullOrEmpty(pagesource1))
{
if (proxyStop)
{
return;
}
addInFinalCheckedProxyist(proxyad, proxyport, proxyusername, proxyPassword, pagesource1);
}
else
{
if (proxyStop)
{
return;
}
DemoStagramPro.ClGlobul.isProxyCheckComplete = true;
LoggerProxy("[ " + DateTime.Now + " ] => [ Proxy Is not Working : " + proxyad + ":" + proxyport + ":" + proxyusername + ":" + proxyPassword + " ]");
lock (lockerforNonWorkingProxies)
{
GramBoardProFileHelper.AppendStringToTextfileNewLine(proxyad + ":" + proxyport + ":" + proxyusername + ":" + proxyPassword, GramBoardProFileHelper.NonWorkingProxiesList);
}
}
}
catch (Exception)
{
if (proxyStop)
{
return;
}
LoggerProxy("[ " + DateTime.Now + " ] => [ Proxy Is not Working : " + proxyad + ":" + proxyport + ":" + proxyusername + ":" + proxyPassword + " ]");
lock (lockerforNonWorkingProxies)
{
GramBoardProFileHelper.AppendStringToTextfileNewLine(proxyad + ":" + proxyport + ":" + proxyusername + ":" + proxyPassword, GramBoardProFileHelper.NonWorkingProxiesList);
}
}
finally
{
lock (lockerforProxies)
{
countParseProxiesThreads--;
Monitor.Pulse(lockerforProxies);
}
Proxystatus--;
if (Proxystatus == 0)
{
LoggerProxy("[ " + DateTime.Now + " ] => [ PROCESS COMPLETED ]");
LoggerProxy("-----------------------------------------------------------------------------------------------------------------------------------");
}
}
}
/// <summary> Get User details for While/blacklist User
/// Get User details for While/blacklist User
/// </summary>
/// <param name="username">User Screan Name OR User ID</param>
/// <returns>Details of Users</returns>
///
#region Get User details for While/blacklist User
//public static Dictionary<string, string> GetUserDetails(string username, out string status)
//{
//// ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
// Dictionary<string, string> dataLst = new Dictionary<string, string>();
//// try
//// {
//// string PageSource = null;
//// if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username))
//// {
//// //PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?user_id=" + username + "&include_entities=true");
//// }
//// else
//// {
//// //PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?screen_name=" + username + "&include_entities=true");
//// string url = string.Format("https://twitter.com/" + username);
//// PageSource = httpHelper.GetHtml(url);
//// }
//// if (PageSource.Contains("error\":\"User has been suspended"))
//// {
//// status = "User has been suspended";
//// return dataLst;
//// }
//// else if (PageSource.Contains("message\":\"Sorry, that page does not exist"))
//// {
//// status = "Sorry, that page does not exist";
//// return dataLst;
//// }
//// else if (PageSource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour."))
//// {
//// status = "Rate limit exceeded. Clients may not make more than 150 requests per hour.";
//// return dataLst;
//// }
//// if (!string.IsNullOrEmpty(PageSource))
//// {
//// string id = string.Empty;
//// string name = string.Empty;
//// string screen_name = string.Empty;
//// string location = string.Empty;
//// string NoOfTweet = string.Empty;
//// string followers_count = string.Empty;
//// string friends_count = string.Empty;
//// string profile_image_url = string.Empty;
//// if (PageSource.Contains("data-user-id="))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "data-user-id=");
//// if (getDataInArr.Count() > 0)
//// {
//// foreach (var item in getDataInArr)
//// {
//// int startindex = item.IndexOf("<strong>");
//// int endindex = item.IndexOf("</strong>"); ;
//// NoOfTweet = item.Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
//// }
//// }
//// }
//// if (PageSource.Contains("tweet_stats"))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "tweet_stats");
//// if (getDataInArr.Count() > 0)
//// {
//// int startindex = getDataInArr[1].IndexOf("<strong>");
//// int endindex = getDataInArr[1].IndexOf("</strong>"); ;
//// NoOfTweet = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
//// }
//// }
//// if (PageSource.Contains("following_stats"))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "following_stats");
//// if (getDataInArr.Count() > 0)
//// {
//// int startindex = getDataInArr[1].IndexOf("<strong>");
//// int endindex = getDataInArr[1].IndexOf("</strong>"); ;
//// friends_count = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
//// }
//// }
//// if (PageSource.Contains("follower_stats"))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "follower_stats");
//// if (getDataInArr.Count() > 0)
//// {
//// int startindex = getDataInArr[1].IndexOf("<strong>");
//// int endindex = getDataInArr[1].IndexOf("</strong>");
//// followers_count = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
//// }
//// }
//// if (PageSource.Contains("data-url"))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "data-url");
//// if (getDataInArr.Count() > 0)
//// {
//// int startindex = getDataInArr[1].IndexOf("=\"");
//// int endindex = getDataInArr[1].IndexOf(">"); ;
//// profile_image_url = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("target=\"_blank", string.Empty).Replace("\"", string.Empty).Replace("=", string.Empty).Trim();
//// }
//// }
//// //string id = jobj["id"].ToString().Replace("\"", string.Empty);
//// dataLst.Add("id", id);
//// //string name = jobj["name"].ToString().Replace("\"", string.Empty);
//// dataLst.Add("name", name);
//// //string screen_name = jobj["screen_name"].ToString().Replace("\"", string.Empty);
//// dataLst.Add("screen_name", screen_name);
//// //string location = jobj["location"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("location", location);
//// //string description = jobj["description"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("description", description);
//// //string followers_count = jobj["followers_count"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("followers_count", followers_count);
//// //string friends_count = jobj["friends_count"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("friends_count", friends_count);
//// //string statuses_count = jobj["statuses_count"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("statuses_count", statuses_count);
//// //string created_at = jobj["created_at"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("created_at", created_at);
//// //string time_zone = jobj["time_zone"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("time_zone", time_zone);
//// //string profile_image_url = jobj["profile_image_url"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("profile_image_url", profile_image_url);
//// }
//// }
//// catch (Exception ex)
//// {
//// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
//// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
//// }
//// //Assign Status
// status = "Ok";
// return dataLst;
//}
public static Dictionary<string, string> GetUserDetails(string username, out string status)
{
TwitterDataScrapper twiterDataScrapper = new TwitterDataScrapper();
Dictionary<string, string> dataLst = new Dictionary<string, string>();
string id = string.Empty;
string name = string.Empty;
string screen_name = string.Empty;
string location = string.Empty;
string description = string.Empty;
string NoOfTweet = string.Empty;
string followers_count = string.Empty;
string following_count = string.Empty;
string profile_image_url = string.Empty;
//string ProfileName = string.Empty;
//string Location = string.Empty;
//string Bio = string.Empty;
string website = string.Empty;
//string NoOfTweets = string.Empty;
//string Followers = string.Empty;
//string Followings = string.Empty;
//try
//{
ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr();
GlobusHttpHelper HttpHelper = new GlobusHttpHelper();
string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + username), "", "");
string Responce = ProfilePageSource;
#region Convert HTML to XML
string xHtml = objChilkat.ConvertHtmlToXml(Responce);
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
//Get User Id From page
if (!(ProfilePageSource == null))
{
if (ProfilePageSource.Contains("data-user-id="))
{
try
{
String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(ProfilePageSource, "data-user-id=");
if (getDataInArr.Count() > 0)
{
foreach (var item in getDataInArr)
{
if (item.Contains("<h1"))
{
int startindex = (1);
int endindex = item.IndexOf(">") - 1; ;
id = item.Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
if (id.Contains("\n"))
{
string[] arr =System.Text.RegularExpressions.Regex.Split(id,"\"");
id=arr[0];
}
dataLst.Add("id", id);
break;
}
}
//whiteListedUserCount++;
}
}
catch (Exception)
{
}
}
}
//else
//{
// blackListedUserCount++;
//}
int counterdata = 0;
xBeginSearchAfter = null;
string dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNameTruncated-link u-textInheritColor js-nav js-action-profile-name");//"profile-field");
while ((xNode != null))
{
try
{
string str = xNode.GetXml();
xBeginSearchAfter = xNode;
if (counterdata == 1)
{
website = xNode.AccumulateTagContent("text", "script|style");
counterdata++;
}
else if (counterdata == 0)
{
name = xNode.AccumulateTagContent("text", "script|style");
dataLst.Add("name", name);
counterdata++;
}
else
{
break;
}
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNameTruncated-link u-textInheritColor js-nav js-action-profile-name");//"profile-field");
}
catch (Exception)
{
}
}
//Get Screen name and ID
xBeginSearchAfter = null;
//string dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "u-linkComplex-target");
while ((xNode != null))
{
string str = xNode.GetXml();
xBeginSearchAfter = xNode;
screen_name = xNode.AccumulateTagContent("text", "script|style").Replace("@", string.Empty);
dataLst.Add("screen_name", screen_name);
break;
}
xBeginSearchAfter = null;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//"location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
location = xNode.AccumulateTagContent("text", "script|style");
dataLst.Add("location", location);
break;
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
description = xNode.AccumulateTagContent("text", "script|style");
dataLst.Add("description", description);
break;
}
int counterData = 0;
xBeginSearchAfter = null;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");
xBeginSearchAfter = xNode;
if (counterData == 0)
{
NoOfTweet = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet",string.Empty);
dataLst.Add("statuses_count", NoOfTweet);
counterData++;
}
//dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
xBeginSearchAfter = xNode;
if (counterData == 1)
{
following_count = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty);
dataLst.Add("friends_count", following_count);
counterData++;
}
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
//while ((xNode != null))
//{
// //xBeginSearchAfter = xNode;
//if (counterData == 0)
//{
// NoOfTweet = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet",string.Empty);
// dataLst.Add("statuses_count", NoOfTweet);
// counterData++;
//}
xBeginSearchAfter = xNode;
if (counterData == 2)
{
followers_count = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty);
dataLst.Add("followers_count", followers_count);
counterData++;
}
//else
//{
// break;
//}
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//"js-nav");
// }
status = "Ok";
//}
//catch (Exception)
//{
// status = "";
//}
return dataLst;
}
//public static int blackListedUserCount = 0;
//public static int whiteListedUserCount = 0;
public static string GetUserIDFromUsername(string username, out string Status)
{
string GetStatus = string.Empty;
Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper();
clsDBQueryManager DB = new clsDBQueryManager();
DataSet ds = DB.GetUserId(username);
string user_id = string.Empty;
try
{
if (ds.Tables[0].Rows.Count > 0)
{
foreach (DataRow dataRow in ds.Tables["tb_UsernameDetails"].Rows)
{
user_id = dataRow.ItemArray[0].ToString();
Status = "No Error";
return user_id;
}
}
}
catch { };
try
{
#region gs httpHelper code
//if (username.Contains("@"))
//{
//}
//string pagesource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?screen_name=" + username), "api.twitter.com", "");
#endregion
string id = string.Empty;
string pagesource = string.Empty;
ChilkatHttpHelpr httpHelper1 = new ChilkatHttpHelpr();
if (NumberHelper.ValidateNumber(username))
{
pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + username);
if (string.IsNullOrEmpty(pagesource))
{
pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + username);
}
}
else
{
pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username);
if (string.IsNullOrEmpty(pagesource))
{
pagesource = httpHelper1.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username);
}
}
if (!pagesource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour.") && !pagesource.Contains("Sorry, that page does not exist") && !pagesource.Contains("User has been suspended"))
{
if (NumberHelper.ValidateNumber(username))
{
int length = pagesource.IndexOf("</screen_name>");
username = pagesource.Substring(pagesource.IndexOf("<screen_name>"), length - pagesource.IndexOf("<screen_name>")).Replace("<screen_name>", "");
user_id = username;
GetStatus = "No Error";
}
else
{
int length = pagesource.IndexOf("</id>");
id = pagesource.Substring(pagesource.IndexOf("<id>"), length - pagesource.IndexOf("<id>")).Replace("<id>", "");
user_id = id;
GetStatus = "No Error";
}
}
else if (pagesource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour."))
{
GetStatus = "Rate limit exceeded";
}
else if (pagesource.Contains("Sorry, that page does not exist"))
{
GetStatus = "Sorry, that page does not exist";
}
else if (pagesource.Contains("User has been suspended"))
{
GetStatus = "User has been suspended";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserIDFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserIDFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
GetStatus = "Error";
}
Status = GetStatus;
return user_id;
}
public static string GetPhotoFromUsername_New(string username)
{
//Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper();
ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
string user_id = string.Empty;
string containsImage = "false";
try
{
string ImageLink = string.Empty;
string PageSource = string.Empty;
if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username))
{
PageSource = httpHelper.GetHtml("https://twitter.com/account/redirect_by_id?id=" + username);
}
else
{
PageSource = httpHelper.GetHtml("https://twitter.com/" + username);
}
if (!string.IsNullOrEmpty(PageSource))
{
try
{
int indexStart = PageSource.IndexOf("profile-picture media-thumbnail");
if (PageSource.Contains("profile-picture media-thumbnail"))
{
string start = PageSource.Substring(indexStart);
int endIndex = start.IndexOf(" </a>");
string end = start.Substring(0, endIndex);
if (!end.Contains(".png"))
{
int StartIndex = end.IndexOf("src=");
string start1 = end.Substring(StartIndex);
int EndIndex = start1.IndexOf(">");
string ImageLinkData = start1.Substring(0, EndIndex);
ImageLink = System.Text.RegularExpressions.Regex.Split(ImageLinkData, "alt")[0].Replace("src=\"", string.Empty).Replace("\"", string.Empty);
}
else
{
ImageLink = string.Empty;
}
}
if (!string.IsNullOrEmpty(ImageLink) && ImageLink.Contains("profile_images"))
{
containsImage = "true";
}
else if (string.IsNullOrEmpty(ImageLink))
{
containsImage = "false";
}
else if (PageSource.Contains("Sorry, that page does not exist"))
{
containsImage = "false";
}
else
{
containsImage = "false";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
else if (PageSource.Contains("Clients may not make more than 150 requests per hour") && PageSource.Contains("Rate limit exceeded"))
{
containsImage = "Rate limit exceeded";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return containsImage;
}
public List<string> GetFollowers(string userID, out string ReturnStatus)
{
string cursor = "-1";
string FollowingUrl = string.Empty;
List<string> lstIds = new List<string>();
try
{
StartAgain:
if (NumberHelper.ValidateNumber(userID))
{
FollowingUrl = "https://api.twitter.com/1/followers/ids.json?cursor=" + cursor + "&id=" + userID + "";//"https://api.twitter.com/1/friends/ids.json?cursor=-1&screen_name=SocioPro";
}
else
{
FollowingUrl = "https://api.twitter.com/1/followers/ids.json?cursor=" + cursor + "&screen_name=" + userID + "";//"https://api.twitter.com/1/friends/ids.json?cursor=-1&screen_name=SocioPro";
}
#region gs http helper code
//https://api.twitter.com/1/following/ids.json?cursor=-1&screen_name=SocioPro
//string Data = globushttpHelper.getHtmlfromUrl(new Uri(FollowingUrl), "", "");
#endregion
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
string Data = HttpHelper.GetHtml(FollowingUrl);
if (Data.Contains("401 Unauthorized"))
{
ReturnStatus = "Account is Suspended. ";
return new List<string>();
}
else if (!Data.Contains("Rate limit exceeded") && !Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}") && !string.IsNullOrEmpty(Data))
{
int FirstPoint = Data.IndexOf("[");
int SecondPoint = Data.IndexOf("]");
string FollowingIds = Data.Substring(FirstPoint, SecondPoint - FirstPoint).Replace("[", string.Empty).Replace("]", string.Empty);
List<string> tempid = FollowingIds.Split(',').ToList();
foreach (string item in tempid)
{
lstIds.Add(item);
}
if (Data.Contains("next_cursor_str"))
{
int startindex = Data.IndexOf("\"next_cursor_str\":");
string start = Data.Substring(startindex).Replace("\"next_cursor_str\":", "");
int lastindex = start.IndexOf("\",\"");
string end = start.Substring(0, lastindex).Replace("\"", "");
cursor = end;
if (cursor != "0")
{
goto StartAgain;
}
}
ReturnStatus = "No Error";
return lstIds;
}
else if (Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}"))
{
ReturnStatus = "Sorry, that page does not exist :" + userID;
return lstIds;
}
else if (Data.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour."))
{
ReturnStatus = "Rate limit exceeded. Clients may not make more than 150 requests per hour.:-" + userID;
return lstIds;
}
else
{
ReturnStatus = "Error";
return lstIds;
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowers() -- " + userID + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowers() -- " + userID + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
ReturnStatus = "Error";
return lstIds;
}
}
public List <string> GetTweetData_Scrape(string keyword, out string returnStatus)
{
string status = string.Empty;
List <string> lst_TweetIDs = new List <string>();
try
{
string user_id = string.Empty;
#region old code
//if (!NumberHelper.ValidateNumber(keyword))
//{
// user_id = GetUserIDFromUsername(keyword);
//}
//else
//{
// user_id = keyword;
//}
//if (!string.IsNullOrEmpty(user_id))
//{
//screen_name
#endregion
string searchURL = string.Empty;
if (NumberHelper.ValidateNumber(keyword))
{
searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + keyword + "&count=" + TweetExtractCount;
}
else
{
searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&screen_name=" + keyword + "&count=" + TweetExtractCount;
}
//string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
string res_Get_searchURL = HttpHelper.GetHtml(searchURL);
if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && !res_Get_searchURL.Contains("Not authorized") && res_Get_searchURL.Contains("created_at") && !string.IsNullOrEmpty(res_Get_searchURL))
{
string[] splitRes = Regex.Split(res_Get_searchURL, "{\"created_at");//Regex.Split(res_Get_searchURL, "{\"created_at\"");
splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
//string text = Globussoft.GlobusHttpHelper.ParseJson(modified_Item, "<text>");
string Tweet = Globussoft.GlobusHttpHelper.ParseJson(item, "text");//Globussoft.GlobusHttpHelper.parseText(item);
string data = keyword + ":" + Tweet;
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(data, Globals.Path_TweetExtractor);
lst_TweetIDs.Add(data);
}
status = "No Error";
}
else if (res_Get_searchURL.Contains("Rate limit exceeded"))
{
status = "Rate limit exceeded";
}
else if (res_Get_searchURL.Contains("Sorry, that page does not exist"))
{
status = "Sorry, that page does not exist";
}
else if (res_Get_searchURL.Contains("Not authorized"))
{
status = "Not Authorized";
}
else if (string.IsNullOrEmpty(res_Get_searchURL))
{
status = "Not Authorized";
}
else
{
status = "Empty";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
status = "Error";
}
returnStatus = status;
return(lst_TweetIDs);
}
public List <string> GetRetweetData_Scrape(string keyword, out string returnStatus)
{
string status = string.Empty;
List <string> lst_ReTweetIDs = new List <string>();
try
{
string searchURL = string.Empty;
if (!NumberHelper.ValidateNumber(keyword))
{
searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?screen_name=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true";
}
else if (NumberHelper.ValidateNumber(keyword))
{
searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?id=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true";
}
//string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
string res_Get_searchURL = HttpHelper.GetHtml(searchURL);
if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && res_Get_searchURL.Contains("created_at") && !string.IsNullOrEmpty(res_Get_searchURL))
{
string[] splitRes = Regex.Split(res_Get_searchURL, "<status>");//Regex.Split(res_Get_searchURL, "{\"created_at\"");
splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
string Tweet = string.Empty;
string Tweeter = string.Empty;
try
{
int startIndex = item.IndexOf("<text>");
string start = item.Substring(startIndex);
int endIndex = start.IndexOf("</text>");
string end = start.Substring(0, endIndex);
Tweet = end.Replace("<text>", "");
int startOfInndex = Tweet.IndexOf(":");
Tweeter = Tweet.Substring(0, startOfInndex);
Tweet = Tweet.Replace(Tweeter, "");
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
string data = keyword + ":" + Tweeter + ":" + Tweet.Replace(":", "^");
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(data, Globals.Path_RETweetExtractor);
lst_ReTweetIDs.Add(data);
}
}
else if (res_Get_searchURL.Contains("Rate limit exceeded"))
{
status = "Rate limit exceeded";
}
else if (res_Get_searchURL.Contains("Sorry, that page does not exist"))
{
status = "Sorry, that page does not exist";
}
else if (res_Get_searchURL.Contains("Not authorized"))
{
status = "Not Authorized";
}
else if (string.IsNullOrEmpty(res_Get_searchURL))
{
status = "Not Authorized";
}
else
{
status = "Empty";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
status = "Error";
}
returnStatus = status;
return(lst_ReTweetIDs);
}
public static string GetPhotoFromUsername(string username)
{
//Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper();
ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
string user_id = string.Empty;
string containsImage = "false";
try
{
string ImageLink = string.Empty;
string PageSource = string.Empty;
//http://a0.twimg.com/sticky/default_profile_images/default_profile_0_normal.png
if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username))
{
//PageSource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?user_id=" + username + "&include_entities=true"), "", "");
PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + username + "&include_entities=true");
}
else
{
//PageSource = httpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/users/show.xml?screen_name=" + username + "&include_entities=true"), "", "");
PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?screen_name=" + username + "&include_entities=true");
}
if (!string.IsNullOrEmpty(PageSource))
{
try
{
int indexStart = PageSource.IndexOf("<profile_image_url>");
if (PageSource.Contains("<profile_image_url>"))
{
string start = PageSource.Substring(indexStart);
int endIndex = start.IndexOf("</profile_image_url>");
string end = start.Substring(0, endIndex);
ImageLink = end.Replace("<profile_image_url>", "");
}
if (!string.IsNullOrEmpty(ImageLink) && ImageLink.Contains("/sticky/default_profile_images/default_profile"))
{
containsImage = "false";
}
else if (PageSource.Contains("Sorry, that page does not exist"))
{
containsImage = "false";
}
else
{
containsImage = "true";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
else if (PageSource.Contains("Clients may not make more than 150 requests per hour") && PageSource.Contains("Rate limit exceeded"))
{
containsImage = "Rate limit exceeded";
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return containsImage;
}
public void PostAddcompanyUrl(object parameter)
{
try
{
try
{
if (IsStop)
{
return;
}
if (!IsStop)
{
lstLinkedinFollowCompanyUrlThraed.Add(Thread.CurrentThread);
lstLinkedinFollowCompanyUrlThraed.Distinct().ToList();
Thread.CurrentThread.IsBackground = true;
}
}
catch
{
}
if (!string.IsNullOrEmpty(txtNumberOfFollowPerAccount.Text) && NumberHelper.ValidateNumber(txtNumberOfFollowPerAccount.Text))
{
LinkedinCompanyFollow.CountPerAccount = Convert.ToInt32(txtNumberOfFollowPerAccount.Text);
}
Array paramsArray = new object[1];
paramsArray = (Array)parameter;
KeyValuePair <string, LinkedInMaster> item = (KeyValuePair <string, LinkedInMaster>)paramsArray.GetValue(0);
LinkedinLogin Login = new LinkedinLogin();
Login.accountUser = item.Key;
Login.accountPass = item.Value._Password;
Login.proxyAddress = item.Value._ProxyAddress;
Login.proxyPort = item.Value._ProxyPort;
Login.proxyUserName = item.Value._ProxyUsername;
Login.proxyPassword = item.Value._ProxyPassword;
GlobusHttpHelper HttpHelper = new GlobusHttpHelper();
ChilkatHttpHelpr ChilkatHttpHelper = new ChilkatHttpHelpr();
LinkedinCompanyFollow obj_FollowCompany = new LinkedinCompanyFollow(Login.accountUser, Login.accountPass, Login.proxyAddress, Login.proxyPort, Login.proxyUserName, Login.proxyPassword);
Login.logger.addToLogger += new EventHandler(logger_SearchFollowCompanyaddToLogger);
int minDelay = 20;
int maxDelay = 25;
if (!string.IsNullOrEmpty(txtFollowCompanyMinDelay.Text) && NumberHelper.ValidateNumber(txtFollowCompanyMinDelay.Text))
{
minDelay = Convert.ToInt32(txtFollowCompanyMinDelay.Text);
}
if (!string.IsNullOrEmpty(txtFollowCompanyMaxDelay.Text) && NumberHelper.ValidateNumber(txtFollowCompanyMaxDelay.Text))
{
maxDelay = Convert.ToInt32(txtFollowCompanyMaxDelay.Text);
}
if (!Login.IsLoggedIn)
{
Login.LoginHttpHelper(ref HttpHelper);
//Login.LoginWithChilkatHelper(ref ChilkatHttpHelper);
}
if (Login.IsLoggedIn)
{
LinkedinCompanyFollow LinkdinFollowComp = new LinkedinCompanyFollow();
//LinkdinFollowComp.logger.addToLogger += new EventHandler(logger_SearchFollowCompanyaddToLogger);
try
{
Dictionary <string, string> Result = new Dictionary <string, string>();
string MessagePosted = obj_FollowCompany.PostAddCompanyUsingUrl(ref HttpHelper, Login.accountUser, minDelay, maxDelay);
}
catch { }
}
}
catch (Exception ex)
{
GlobusFileHelper.AppendStringToTextfileNewLine("DateTime :- " + DateTime.Now + " :: Error --> Manage Follow Company URL --> PostAddcompanyUrl() >>>> " + ex.Message + "StackTrace --> >>>" + ex.StackTrace, Globals.Path_LinkedinErrorLogs);
GlobusFileHelper.AppendStringToTextfileNewLine("DateTime :- " + DateTime.Now + " :: Error --> Manage Follow Company URL --> PostAddcompanyUrl() >>>> " + ex.Message + "StackTrace --> >>>" + ex.StackTrace, Globals.Path_LinkedinAddSearchGroupErrorLogs);
}
finally
{
counter_AddFollowUrl--;
if (counter_AddFollowUrl == 0)
{
btnFollowCompany.Invoke(new MethodInvoker(delegate
{
AddLoggerFollowCompanyUrl("[ " + DateTime.Now + " ] => [ PROCESS COMPLETED ]");
AddLoggerFollowCompanyUrl("----------------------------------------------------------------------------------------------------------");
btnFollowCompany.Cursor = Cursors.Default;
}));
}
}
}
public List<string> GetFollowings(string userID, out string ReturnStatus)
{
try
{
string cursor = "-1";
string FollowingUrl = string.Empty;
StartAgain:
if (NumberHelper.ValidateNumber(userID))
{
FollowingUrl = "https://api.twitter.com/1/following/ids.json?cursor=" + cursor + "&id=" + userID + "";//"https://api.twitter.com/1/following/ids.json?cursor=-1&screen_name=SocioPro";
}
else
{
FollowingUrl = "https://api.twitter.com/1/following/ids.json?cursor=" + cursor + "&screen_name=" + userID + "";//"https://api.twitter.com/1/following/ids.json?cursor=-1&screen_name=SocioPro";
}
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
string Data = HttpHelper.GetHtml(FollowingUrl);//.getHtmlfromUrl(new Uri(FollowingUrl), "", "");
if (Data.Contains("that page does not exist"))
{
ReturnStatus = "that page does not exist";
return new List<string>();
}
else if (!Data.Contains("Rate limit exceeded"))
{
int FirstPoint = Data.IndexOf("[");
int SecondPoint = Data.IndexOf("]");
string FollowingIds = Data.Substring(FirstPoint, SecondPoint - FirstPoint).Replace("[", string.Empty).Replace("]", string.Empty);
List<string> lstIds = FollowingIds.Split(',').ToList();
if (Data.Contains("next_cursor_str"))
{
int startindex = Data.IndexOf("\"next_cursor_str\":");
string start = Data.Substring(startindex).Replace("\"next_cursor_str\":", "");
int lastindex = start.IndexOf("\",\"");
string end = start.Substring(0, lastindex).Replace("\"", "");
cursor = end;
if (cursor != "0")
{
goto StartAgain;
}
}
ReturnStatus = "No Error";
return lstIds;
}
else
{
ReturnStatus = "Error";
return new List<string>();
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings() -- " + userID + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings() -- " + userID + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
ReturnStatus = "Error";
return new List<string>();
}
}
public static void RequestJSCSSIMG(string pageSource, ref Chilkat.Http http)
{
ServicePointManager.ServerCertificateValidationCallback = delegate { return(true); };
List <string> listURLs = new List <string>();
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
//CSS Request
foreach (string item in GetHrefsFromString(pageSource))
{
if (item.Contains(".css"))
{
string cssSource = item.Replace(" ", "").Trim();
try
{
//string res = HttpHelper.GetHtmlProxy(cssSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
listURLs.Add(cssSource);
}
catch (Exception)
{
Thread.Sleep(500);
try
{
string res = HttpHelper.GetHtmlProxy(cssSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
}
catch (Exception)
{
}
}
}
}
//JS Request
string[] scriptArr = Regex.Split(pageSource, "/script>");
foreach (string item in scriptArr)
{
try
{
if (item.Contains("static.ak.") || item.Contains("profile.ak."))
{
int startIndx = item.LastIndexOf("src=") + "src=".Length + 1;
int endIndx = item.IndexOf(">", startIndx) - 1;
string jsSource = item.Substring(startIndx, endIndx - startIndx);
//if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak."))
if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.") || jsSource.StartsWith("http://profile.ak.") || jsSource.StartsWith("https://profile.ak.") || jsSource.StartsWith("http://s-profile.ak.") || jsSource.StartsWith("https://s-profile.ak."))
{
try
{
//string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
listURLs.Add(jsSource);
}
catch (Exception)
{
Thread.Sleep(500);
try
{
string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
}
catch (Exception)
{
}
}
}
}
}
catch (Exception)
{
}
}
string[] moreScriptArray = Regex.Split(pageSource, "\"src\":");
foreach (string item in moreScriptArray)
{
try
{
int startIndx = 1;
int endIndx = item.IndexOf("\"", startIndx);
string jsSource = item.Substring(startIndx, endIndx - startIndx).Replace("\\", "");
if (jsSource.Contains(".js"))
{
//string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
listURLs.Add(jsSource);
}
}
catch (Exception)
{
}
}
///IMG Request
string[] imageArr = Regex.Split(pageSource, "<img");
foreach (string item in imageArr)
{
try
{
if (item.Contains("static.ak.") || item.Contains("profile.ak."))
{
int startIndx = item.IndexOf("src=") + "src=".Length + 1;
int endIndx = item.IndexOf("\"", startIndx + 1);
string jsSource = item.Substring(startIndx, endIndx - startIndx);
//if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak."))
if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.") || jsSource.StartsWith("http://profile.ak.") || jsSource.StartsWith("https://profile.ak.") || jsSource.StartsWith("http://s-profile.ak.") || jsSource.StartsWith("https://s-profile.ak."))
{
if (jsSource.Contains(".png") || jsSource.Contains(".gif") || jsSource.Contains(".jpg") || jsSource.Contains(".jpeg"))
{
try
{
//string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
listURLs.Add(jsSource);
}
catch (Exception)
{
Thread.Sleep(500);
try
{
string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
}
catch (Exception)
{
}
}
}
}
}
}
catch { };
}
listURLs = listURLs.Distinct().ToList();
foreach (string item in listURLs)
{
try
{
string res = HttpHelper.GetHtmlProxy(item, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);
}
catch { };
}
}
//public List<string> GetFollowings_New(string userID, out string ReturnStatus, ref Globussoft.GlobusHttpHelper HttpHelper)
//{
// Log("[ " + DateTime.Now + " ] => [ Searching For Following For " + userID + " ]");
// string cursor = "-1";
// string FollowingUrl = string.Empty;
// List<string> lstIds = new List<string>();
// try
// {
// StartAgain:
// string Data = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + userID + "/following/users?cursor=" + cursor + "&include_available_features=1&include_entities=1&is_forward=true"), "", "");
// if (string.IsNullOrEmpty(Data))
// {
// for (int i = 1; i <= 3; i++)
// {
// Thread.Sleep(3000);
// Data = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + userID + "/followers/users?cursor=" + cursor + "&include_available_features=1&include_entities=1&is_forward=true"), "", "");
// if (!string.IsNullOrEmpty(Data))
// {
// break;
// }
// }
// }
// if (Data.Contains("401 Unauthorized"))
// {
// ReturnStatus = "Account is Suspended. ";
// return lstIds;
// }
// else if (!Data.Contains("Rate limit exceeded") && !Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}") && !string.IsNullOrEmpty(Data))
// {
// //string[] arraydata = Regex.Split(Data, "js-stream-item stream-item stream-item");
// string[] arraydata;
// string startWitn = string.Empty;
// if (Data.Contains("js-stream-item stream-item stream-item"))
// {
// arraydata = Regex.Split(Data, "js-stream-item stream-item stream-item");
// startWitn = "";
// }
// else
// {
// arraydata = Regex.Split(Data, "js-stream-item");
// startWitn = "\\\" role=\\\"listitem\\\"";
// }
// arraydata = arraydata.Skip(1).ToArray();
// foreach (string id in arraydata)
// {
// string userid = string.Empty;
// string username = string.Empty;
// if (!id.StartsWith(startWitn))
// {
// continue;
// }
// try
// {
// int startindex = id.IndexOf("data-item-id=\\\"");
// string start = id.Substring(startindex).Replace("data-item-id=\\\"", "");
// int endindex = start.IndexOf("\\\"");
// string end = start.Substring(0, endindex);
// userid = end;
// }
// catch (Exception ex)
// {
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings_New() -- " + userID + " --> userid --> " + ex.Message, Globals.Path_TwitterDataScrapper);
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings_New() -- " + userID + " --> userid --> " + ex.Message, Globals.Path_TwtErrorLogs);
// }
// try
// {
// int startindex = id.IndexOf("data-screen-name=\\\"");
// string start = id.Substring(startindex).Replace("data-screen-name=\\\"", "");
// int endindex = start.IndexOf("\\\"");
// string end = start.Substring(0, endindex);
// username = end;
// }
// catch (Exception ex)
// {
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings_New() -- " + userID + " --> username --> " + ex.Message, Globals.Path_TwitterDataScrapper);
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings_New() -- " + userID + " --> username --> " + ex.Message, Globals.Path_TwtErrorLogs);
// }
// if (CounterDataNo > 0)
// {
// if (lstIds.Count == CounterDataNo)
// {
// ReturnStatus = "No Error";
// lstIds = lstIds.Distinct().ToList();
// return lstIds;
// }
// else
// {
// lstIds.Add(userid + ":" + username);
// Globals.lstScrapedUserIDs.Add(userid);
// Log("[ " + DateTime.Now + " ] => [ " + username + ":" + userid + " ]");
// GlobusFileHelper.AppendStringToTextfileNewLine(userID + "," + userid + "," + username, Globals.Path_ScrapedFollowingsList);
// }
// }
// try
// {
// //if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(userid))
// //{
// // string query = "INSERT INTO tb_UsernameDetails (Username , Userid) VALUES ('" + username + "' ,'" + userid + "') ";
// // DataBaseHandler.InsertQuery(query, "tb_UsernameDetails");
// //}
// }
// catch (Exception ex)
// {
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings_New() -- " + userID + " --> Database --> " + ex.Message, Globals.Path_TwitterDataScrapper);
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings_New() -- " + userID + " --> DataBase --> " + ex.Message, Globals.Path_TwtErrorLogs);
// }
// }
// if (Data.Contains("\"has_more_items\":true"))
// {
// int startindex = Data.IndexOf("\"cursor\":");
// string start = Data.Substring(startindex).Replace("\"cursor\":", "");
// int lastindex = start.IndexOf("\",\"");
// if (lastindex < 0)
// {
// lastindex = start.IndexOf("\"}");
// }
// string end = start.Substring(0, lastindex).Replace("\"", "");
// cursor = end;
// if (cursor != "0")
// {
// if (NoOfFollowingsToBeunfollowed > 0)
// {
// lstIds = lstIds.Distinct().ToList();
// //commented by prabhat
// //if (NoOfFollowingsToBeunfollowed == lst_structTweetIDs.Count)
// if (NoOfFollowingsToBeunfollowed == lstIds.Count)
// {
// ReturnStatus = "No Error";
// return lstIds;
// }
// }
// goto StartAgain;
// }
// }
// ReturnStatus = "No Error";
// lstIds = lstIds.Distinct().ToList();
// return lstIds;
// }
// else if (Data.Contains("{\"errors\":[{\"message\":\"Sorry, that page does not exist\",\"code\":34}]}"))
// {
// ReturnStatus = "Sorry, that page does not exist :" + userID;
// lstIds = lstIds.Distinct().ToList();
// return lstIds;
// }
// else if (Data.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour."))
// {
// ReturnStatus = "Rate limit exceeded. Clients may not make more than 150 requests per hour.:-" + userID;
// lstIds = lstIds.Distinct().ToList();
// return lstIds;
// }
// else
// {
// ReturnStatus = "Error";
// lstIds = lstIds.Distinct().ToList();
// return lstIds;
// }
// }
// catch (Exception ex)
// {
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetFollowings_New() -- " + userID + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetFollowings_New() -- " + userID + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
// ReturnStatus = "Error";
// lstIds = lstIds.Distinct().ToList();
// return lstIds;
// }
//}
public List<string> GetHashTags(out string returnStatus)
{
List<string> HashTags = new List<string>();
try
{
//string pagesource = globushttpHelper.getHtmlfromUrl(new Uri("https://api.twitter.com/1/trends/daily.json"), "", "");
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
string pagesource = HttpHelper.GetHtml("https://api.twitter.com/1/trends/daily.json");
if (!pagesource.Contains("Rate limit exceeded"))
{
string[] array = Regex.Split(pagesource, "\"name\":");
array = array.Skip(1).ToArray();
foreach (string item in array)
{
try
{
int startindex = item.IndexOf("\"");
string Start = item.Substring(startindex);
int endIndex = Start.IndexOf("\",");
string End = Start.Substring(0, endIndex).Replace("\"", "");
string HashTag = End;
if (HashTag.Contains("#"))
{
HashTags.Add(HashTag);
}
}
catch (Exception ex)
{
returnStatus = "Error";
return new List<string>();
}
}
returnStatus = "No Error";
return HashTags;
}
else
{
returnStatus = "Error";
return new List<string>();
}
}
catch (Exception ex)
{
returnStatus = "Error";
return new List<string>();
}
}
public void DataParser(string response, ref FacebookUser fbUser)
{
try
{
try
{
string Response = response;
ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr();
string FirstPgaeName = string.Empty;
string SecondPgaeName = string.Empty;
string otherPgaeName = string.Empty;
string PageTitle = string.Empty;
string PageTitleLink = string.Empty;
string SourceTitle = string.Empty;
string SourceTitleLink = string.Empty;
string memberCount = string.Empty;
string FirstPgaeNameLink = string.Empty;
string SecondPgaeNameLink = string.Empty;
string otherPgaeNameLink = string.Empty;
string likerData = string.Empty;
string SecondlikerData = string.Empty;
string ThirdlikerData = string.Empty;
string AboutData = string.Empty;
string AboutDataLink = string.Empty;
bool MemberPage = false;
string[] DataList = System.Text.RegularExpressions.Regex.Split(Response, "<div class=\"");
foreach (string item2 in DataList)
{
#region Title
if (item2.Contains(""title""))
{
List <string> hrefLink = objChilkat.GetHrefFromString(item2);
try
{
string titleData = item2.Substring(item2.IndexOf("\"><a") + 5);
string DataPage = GetSbstringData(titleData, "\">", "<");
PageTitle = DataPage;
PageTitleLink = hrefLink[0].ToString();
}
catch { };
}
#endregion
#region GroupSource
if (item2.Contains("quot;sub_headers""))
{
List <string> hrefLink = objChilkat.GetHrefFromString(item2);
try
{
string titleData = item2.Substring(item2.IndexOf("\"><a") + 5);
string DataPage = GetSbstringData(titleData, "\">", "<");
SourceTitle = DataPage;
if (hrefLink.Count() > 0)
{
SourceTitleLink = hrefLink[0].ToString();
}
}
catch { };
}
#endregion
# region Members
if (item2.Contains("members") && item2.Count() < 150 && MemberPage == false)
{
MemberPage = true;
try
{
string MemberData = GetSbstringData(item2, "\">", "<").Replace("members", "").Replace(",", "");
memberCount = MemberData;
}
catch { };
}
#endregion
#region Aboutdata
#endregion
#region Friendslink
#endregion
}
string Username = string.Empty;
if (!string.IsNullOrEmpty(PageTitle))
{
string data = PageTitleLink.Substring(0, PageTitleLink.IndexOf("?"));
Username = data.Split('/')[3];
// Username = PageTitle + " : " + PageTitleLink;
}
string Likes = string.Empty;
if (!string.IsNullOrEmpty(FirstPgaeName))
{
//Likes ="Likes "+ FirstPgaeName + " : " + FirstPgaeNameLink;
Likes = "Likes " + FirstPgaeNameLink;
}
if (!string.IsNullOrEmpty(SecondPgaeName))
{
// Likes = Likes + " and " + SecondPgaeName + " : " + SecondPgaeNameLink;
Likes = Likes + " and " + SecondPgaeNameLink;
}
if (!string.IsNullOrEmpty(otherPgaeName))
{
// Likes = Likes + " and " + otherPgaeName + " : " + otherPgaeNameLink;
Likes = Likes + " and " + otherPgaeNameLink;
}
if (Username.Count() > 5)
{
try
{
string FollowerData = string.Empty;
string Userdata = Username.Replace(",", "") + "," + PageTitle.Replace(",", "") + "," + SourceTitle.Replace(",", "") + "," + memberCount.Replace(",", "") + "," + AboutData.Replace(",", "");
string FileData = Environment.GetFolderPath(Environment.SpecialFolder.Desktop) + "\\FBScraperDataDirectory\\SearchUrlUserId.txt";
if (!File.Exists(FileData))
{
try
{
string DataHeader = " UserId " + "\t" + " Likes " + "\t" + " Studied In " + " \\t " + " Listens " + " \\t " + "Followers" + "\\t " + "Watches";
GlobusFileHelper.AppendStringToTextfileNewLine(DataHeader, exportFilePathAccountVerification + "\\GroupSearchData");
}
catch { };
}
GlobusFileHelper.AppendStringToTextfileNewLine(Userdata, exportFilePathAccountVerification + "\\GroupSearchData");
//AppendStringToTextFileNewLine(Data, FileData);
}
catch { };
}
}
catch { };
}
/// <summary>
/// Checks the status of the CPUID from Database
/// If status is Active, MainFrm starts
/// </summary>
public bool ValidateCPUID(ref string statusMessage, string servr, ref string username, ref string Password, ref string txnID, string freeTrialKey, string cpuID, ref string Email)
{
//string cpuID = getCPUID();
try
{
#region Drct
//string cpuID = FetchMacId();
//string SelectQuery = "Select * from users where cpuid='" + cpuID + "'";
//DataSet ds = DataBaseHandler.SelectQuery(SelectQuery, "users");
//if (ds.Tables[0].Rows.Count == 1)
//{
// string status = ds.Tables[0].Rows[0]["status"].ToString();
// if (status.ToLower() == "active")
// {
// statusMessage = "active";
// return true;
// }
// else if (status.ToLower() == "nonactive")
// {
// statusMessage = "nonactive";
// return false;
// }
// else if (status.ToLower() == "suspended")
// {
// statusMessage = "suspended";
// return false;
// }
//}
#endregion
#region Through php
//string cpuID = FetchMacId();
//ChilkatHttpHelpr HttpHelpr = new ChilkatHttpHelpr();
HttpHelpr = new ChilkatHttpHelpr();
#region Servr 1
{
string res = string.Empty;
res = HttpHelpr.GetHtml("http://" + servr + "/GetUserData.php?cpid=" + cpuID + "");
if (string.IsNullOrEmpty(res))
{
System.Threading.Thread.Sleep(1000);
res = HttpHelpr.GetHtml("http://" + servr + "/releases/GetUserData.php?cpid=" + cpuID + "");
}
if (!string.IsNullOrEmpty(res))
{
string activationstatus = string.Empty;
string dateTime = string.Empty;
string trimmed_response = res.Replace("<pre>", "").Replace("</pre>", "").Trim().ToLower();
string[] array_status = System.Text.RegularExpressions.Regex.Split(trimmed_response, "<:>");
try
{
activationstatus = array_status[0].ToLower();
}
catch { }
try
{
dateTime = array_status[1].ToLower();
}
catch { }
try
{
username = array_status[2].ToLower();
}
catch { }
try
{
Password = array_status[3].ToLower();
}
catch { };
try
{
txnID = array_status[4].ToLower();
}
catch { }
try
{
Email = array_status[5].ToLower();
}
catch { };
try
{
Globals.licType = array_status[6].ToLower();
}
catch { };
try
{
if (txnID.Contains("freetrial"))
{
Globals.Licence_Details = username + ":" + dateTime + "&" + "FreeTrial" + "";
}
else
{
Globals.Licence_Details = username + ":" + dateTime + "&" + "Full Version" + "";
}
}
catch { }
if (trimmed_response.ToLower().Contains(freeTrialKey) && ((activationstatus.ToLower() == "active") || (activationstatus.ToLower() == "nonactive")))
{
PDGlobals.Instance.isfreeversion = true;
if (CheckActivationUpdateStatus(cpuID, dateTime, activationstatus, servr))
{
statusMessage = "active";
return(true);
}
else
{
statusMessage = "trialexpired";
return(false);
}
if (activationstatus.ToLower() == "active")
{
statusMessage = "active";
return(true);
}
else//else if (activationstatus.ToLower() == "nonactive")
{
//Update status as Active
string updateRes = HttpHelpr.GetHtml("http://" + servr + "/UpdateStatus.php?cpid=" + cpuID + "&status=" + "Active");
if (string.IsNullOrEmpty(updateRes))
{
System.Threading.Thread.Sleep(1000);
updateRes = HttpHelpr.GetHtml("http://" + servr + "/UpdateStatus.php?cpid=" + cpuID + "&status=" + "Active");
}
MessageBox.Show("Your Free Version is Activated");
return(true);
}
}
else if (activationstatus.ToLower() == "active")
{
statusMessage = "active";
return(true);
// DisableControls();
}
else if (activationstatus.ToLower() == "nonactive")
{
statusMessage = "nonactive";
MessageBox.Show("Verification of your txn is under process.\n Please wait for your Transaction to be verified.\n Please Contact To Support Team to activate your license, Skype Id Is :- Facedominatorsupport");
return(false);
//DisableControls();
}
else if (trimmed_response.Contains("trialexpired"))
{
statusMessage = "trialexpired";
MessageBox.Show("Your 3 Days Trial Version has Expired. Please visit our site: facedominator.com to purchase your License");
return(false);
}
else if (trimmed_response.ToLower() == "suspended")
{
statusMessage = "suspended";
return(false);
}
else if (trimmed_response.Contains("no record found"))
{
statusMessage = "norecordfound";
return(false);
}
else
{
statusMessage = "Some Error in Licensing Server";
return(false);
}
}
else
{
statusMessage = "ServerDown";
return(false);
}
}
#endregion
#endregion
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
statusMessage = "Error in License Validation";
MessageBox.Show(ex.StackTrace);
}
return(false);
}
private void ScrapeWhotoFollow(TweetAccountManager tweetAccountManager, string Keyword, int NoOfPages)
{
try
{
AddThreadToDictionary(strModule(Module.WhoToScrap), tweetAccountManager.Username);
int counter = 0;
int PageCount = 1;
// int NoOfData = NoOfPages * 20;
List<string> username = new List<string>();
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Getting Users To Scrape Data For ]");
while (counter < NoOfPages)
{
// string pagsource = tweetAccountManager.globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/who_to_follow/suggestions/search/users?q=" + Keyword + "&cursor=" + PageCount + "&include_available_features=1&include_entities=1&is_forward=true"), "", "");
string pagsource = "";
if (PageCount == 1)
{
pagsource = tweetAccountManager.globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/search/timeline?q=" + Keyword + "&cursor=" + PageCount + "&include_available_features=1&include_entities=1&is_forward=true"), "", ""); // https://twitter.com/who_to_follow/suggestions/search/users?q=software&cursor=1&include_available_features=1&include_entities=1&is_forward=true
}
else
{
//if (pagsource.Contains("\"has_more_items\":true"))
string uri = "https://twitter.com/i/search/timeline?q=" + Keyword + "&mode=users&include_available_features=1&include_entities=1&last_note_ts=555&scroll_cursor=USER-0-" + PageCount * 20;
pagsource = tweetAccountManager.globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/search/timeline?q=" + Keyword + "&mode=users&include_available_features=1&include_entities=1&last_note_ts=555&scroll_cursor=USER-0-" + PageCount * 20), "", "");
}
// if (pagsource.Contains("has-more-items"))
if (pagsource.Contains("has_more_items"))
{
PageCount++;
string[] Aray = Regex.Split(pagsource, "js-stream-item stream-item");
//string[] Aray = Regex.Split(pagsource, "js-stream-item stream-item stream-user-item"); //js-stream-item stream-item
// string[] Aray = Regex.Split(pagsource, "fullname js-action-profile-name");
Aray = Aray.Skip(1).ToArray();
foreach (string item in Aray)
{
string Userid = string.Empty;
string Username = string.Empty;
try
{
//int startindex = item.IndexOf("=\\\"");
int startindex = item.IndexOf("data-user-id=\"");
int startindexForItem = item.IndexOf("data-item-id=\\\"");
if (startindex >= 0 && PageCount==2)
{
string start = item.Substring(startindex).Replace("data-user-id=\"", "");
//string start = item.Substring(startindex).Replace("=\\\"", "");
//int endindex = start.IndexOf("\\\"");
int endindex = start.IndexOf("\"");
string end = start.Substring(0, endindex);
Userid = end;
}
else if (startindexForItem >= 0)
{
string start = item.Substring(startindexForItem).Replace("data-item-id=\\\"", "");
//string start = item.Substring(startindex).Replace("=\\\"", "");
//int endindex = start.IndexOf("\\\"");
int endindex = start.IndexOf("\\\"");
string end = start.Substring(0, endindex);
Userid = end;
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_TwtErrorLogs);
}
try
{
//int startindex = item.IndexOf("data-screen-name=\\\"");
int startindex = item.IndexOf("data-screen-name=\"");
int startindex2 = item.IndexOf("data-screen-name=\\\"");
if (startindex >= 0)
{
//string start = item.Substring(startindex).Replace("data-screen-name=\\\"", "");
string start = item.Substring(startindex).Replace("data-screen-name=\"", "");
//int endindex = start.IndexOf("\\\"");
int endindex = start.IndexOf("\"");
string end = start.Substring(0, endindex);
Username = end;
username.Add(end);
}
if (startindex2 >= 0)
{
//string start = item.Substring(startindex).Replace("data-screen-name=\\\"", "");
string start = item.Substring(startindex2).Replace("data-screen-name=\\\"", "");
//int endindex = start.IndexOf("\\\"");
int endindex = start.IndexOf("\\\"");
string end = start.Substring(0, endindex);
Username = end;
username.Add(end);
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_TwtErrorLogs);
}
try
{
if (!string.IsNullOrEmpty(Username) && !string.IsNullOrEmpty(Userid))
{
string query = "INSERT INTO tb_UsernameDetails (Username , Userid) VALUES ('" + Username + "' ,'" + Userid + "') ";
DataBaseHandler.InsertQuery(query, "tb_UsernameDetails");
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() --> Database --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() --> DataBase --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + Username + " :: " + Userid + " ]");
}
username = username.Distinct().ToList();
counter++;
}
else
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ No More Pages To Scrape For Keyword : " + Keyword + " ]");
break;
}
}
if (!File.Exists(Globals.Path_KeywordScrapedListData + ".csv"))
{
GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NOOFTWEETS , FOLLOWING , FOLLOWERS ", Globals.Path_KeywordScrapedListData + ".csv");
}
foreach (string UserIds in username)
{
try
{
string ProfileName = string.Empty;
string Location = string.Empty;
string Bio = string.Empty;
string website = string.Empty;
string NoOfTweets = string.Empty;
string Followers = string.Empty;
string Followings = string.Empty;
string userids = string.Empty;
string TweetUsername = string.Empty;
string Userid = string.Empty;
string Username = string.Empty;
ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr();
GlobusHttpHelper HttpHelper = new GlobusHttpHelper();
string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + UserIds), "", "");
string Responce = ProfilePageSource;
#region Convert HTML to XML
string xHtml = objChilkat.ConvertHtmlToXml(Responce);
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "ul", "class", "stats js-mini-profile-stats");
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field");
//userids = xNode.GetAttrValue("data-user-id");
//userids = xNode.AccumulateTagContent("text", "script|style");
if (Responce.Contains("has-more-items"))
{
PageCount++;
//string[] Aray = Regex.Split(Responce, "js-stream-item stream-item stream-item");
//Aray = Aray.Skip(1).ToArray();
//foreach (string item in Aray)
{
try
{
#region commented
//int startindex = item.IndexOf("=\\\"");
//int startindex = item.IndexOf("data-user-id=");
//if (startindex >= 0)
//{
// string start = item.Substring(startindex).Replace("data-user-id=\"", "");
// //string start = item.Substring(startindex).Replace("=\\\"", "");
// //int endindex = start.IndexOf("\\\"");
// int endindex = start.IndexOf("\"");
// string end = start.Substring(0, endindex);
// userids = end;
// break;
//}
#endregion
int startindex = Responce.IndexOf("profile_id");
string start = Responce.Substring(startindex).Replace("profile_id", "");
int endindex = start.IndexOf(",");
string end = start.Substring(0, endindex).Replace(""", "").Replace("\"", "").Replace(":", "").Trim();
userids = end.Trim();
TweetUsername = UserIds;
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 1--> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
}
int counterdata = 0;
xBeginSearchAfter = null;
string dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterdata == 0)
{
ProfileName = xNode.AccumulateTagContent("text", "script|style").Replace("Verified account", "");
counterdata++;
}
else if (counterdata == 1)
{
website = xNode.AccumulateTagContent("text", "script|style");
counterdata++;
}
else
{
break;
}
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor");
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "screen-name");
//while ((xNode != null))
//{
// xBeginSearchAfter = xNode;
// TweetUsername = xNode.AccumulateTagContent("text", "script|style");
// break;
//}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim();
break;
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "location profile-field");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Location = xNode.AccumulateTagContent("text", "script|style");
break;
}
int counterData = 0;
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim();
break;
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "location profile-field");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Location = xNode.AccumulateTagContent("text", "script|style");
break;
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "data-element-term", "tweet_stats");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterData == 0)
{
NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty);
counterData++;
}
else if (counterData == 1)
{
Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty);
counterData++;
}
else if (counterData == 2)
{
Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty);
counterData++;
}
else
{
break;
}
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
}
if (!string.IsNullOrEmpty(userids))
{
lock (WhoTofollowThreadLock)
{
GlobusFileHelper.AppendStringToTextfileNewLine(userids + "," + TweetUsername + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + ".csv");
}
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + userids + "," + TweetUsername + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]");
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 3--> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 3--> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Finished Scraping For " + tweetAccountManager.Username + " ]");
AddToProxysLogs("-----------------------------------------------------------------------------------------------------------------------");
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeWhotoFollow() 2--> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeWhotoFollow() 2--> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
public bool CheckPvtIP()
{
try
{
int Working = 0;
string LoggedInIp = string.Empty;
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
GlobusHttpHelper HttpHelper1 = new GlobusHttpHelper();
//string pageSource = HttpHelper.getHtmlfromUrlIPChecker("https://twitter.com/", IPAddress, IPPort, IPUsername, IPpassword);
string pageSource = HttpHelper1.getHtmlfromUrlIPChecker(new Uri("https://twitter.com/"), IPAddress, int.Parse(IPPort), IPUsername, IPpassword);
if (string.IsNullOrEmpty(pageSource))//(string.IsNullOrEmpty(pageSource) && string.IsNullOrEmpty(PgSrcHome))
{
Thread.Sleep(500);
pageSource = HttpHelper1.getHtmlfromUrlIPChecker(new Uri("https://twitter.com/"), IPAddress, int.Parse(IPPort), IPUsername, IPpassword);
if (string.IsNullOrEmpty(pageSource))
{
return false;
}
}
///Logic to check...
//if (pageSource.Contains("class=\"signin\"") && pageSource.Contains("class=\"signup\"") && pageSource.Contains("Twitter"))
if (pageSource.Contains("Sign in") || pageSource.Contains("Sign up") && pageSource.Contains("Twitter"))
{
try
{
using (SQLiteConnection con = new SQLiteConnection(DataBaseHandler.CONstr))
{
using (SQLiteDataAdapter ad = new SQLiteDataAdapter())
{
Working = 1;
string query = "Select * from tb_IP WHERE IPAddress = '" + IPAddress + "' AND IPPort = '" + IPPort + "' AND IPUsername = '******' AND IPpassword = '******' ";
System.Data.DataSet ds = DataBaseHandler.SelectQuery(query, "tb_IP");
if (ds.Tables["tb_IP"].Rows.Count > 0)
{
string UpdateQuery = "UPDATE tb_IP SET Working = 1 WHERE IPAddress = '" + IPAddress + "' AND IPPort = '" + IPPort + "' AND IPUsername = '******' AND IPpassword = '******' ";
DataBaseHandler.UpdateQuery(UpdateQuery, "tb_IP");
}
else
{
string UpdateQuery = "INSERT INTO tb_IP VALUES ('" + IPAddress + "', '" + IPPort + "', '" + IPUsername + "','" + IPpassword + "', 1 , 1 , '" + "" + "' ) ";
DataBaseHandler.InsertQuery(UpdateQuery, "tb_IP");
}
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(IPAddress + ":" + IPPort + ":" + IPUsername + ":" +IPpassword, Globals.Path_ExsistingPvtProxies);
}
}
}
catch (Exception ex)
{
GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> CheckPvtIP() 1 --> " + ex.Message, Globals.Path_IPSettingErroLog);
GlobusFileHelper.AppendStringToTextfileNewLine("Error --> CheckPvtIP() 1 --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return true;
}
return false;
}
catch (Exception ex)
{
GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> CheckPvtIP() 2 --> " + ex.Message, Globals.Path_IPSettingErroLog);
GlobusFileHelper.AppendStringToTextfileNewLine("Error --> CheckPvtIP() 2--> " + ex.Message, Globals.Path_TwtErrorLogs);
return false;
}
}
public static List <string> GetAllFriends(ref ChilkatHttpHelpr chilkatHttpHelper, string userId)
{
List <string> finalList_Friends = new List <string>();
List <string> list_finalFriendsID = new List <string>();
try
{
string pgSource_Friends = chilkatHttpHelper.GetHtml("http://www.facebook.com/profile.php?id=" + userId + "&sk=friends");
string[] Friends = Regex.Split(pgSource_Friends, "user.php");
ParseFriendIDs(Friends, finalList_Friends);
List <string> lstnewfriendid = new List <string>();
// {"collection_token":"1220529617:2356318349:2","cursor":"MDpub3Rfc3RydWN0dXJlZDoxMDAwMDA5MjM1MTg5MjY=","tab_key":"friends","profile_id":1220529617,"overview":false,"ftid":null,"order":null,"sk":"friends"}
string collection_token = "";
string cursor = "";
//check if all friends loaded
string patternAllFriendsLoaded = "\"TimelineAppCollection\",\"setFullyLoaded\",[],[\"pagelet_timeline_app_collection_";
int tempCount_AjaxRequests = 0;
do //
{
try
{
collection_token = "";
cursor = "";
string[] arry = Regex.Split(pgSource_Friends, "enableContentLoader");
if (arry.Length > 1)
{
try
{
string rawData = arry[1];
int startIndx_collection_token = rawData.IndexOf("pagelet_timeline_app_collection_") + "pagelet_timeline_app_collection_".Length;
int endIndx_collection_token = rawData.IndexOf("\"", startIndx_collection_token);
collection_token = rawData.Substring(startIndx_collection_token, endIndx_collection_token - startIndx_collection_token);
int startIndx_cursor = rawData.IndexOf(",\"", endIndx_collection_token) + ",\"".Length;
int endIndx_cursor = rawData.IndexOf("\"", startIndx_cursor);
cursor = rawData.Substring(startIndx_cursor, endIndx_cursor - startIndx_cursor);
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error : " + ex.StackTrace);
}
}
string raw_data = "{\"collection_token\":\"" + collection_token + "\",\"cursor\":\"" + cursor + "\",\"tab_key\":\"friends\",\"profile_id\":" + userId + ",\"overview\":false,\"ftid\":null,\"order\":null,\"sk\":\"friends\"}";
string encoded_raw_data = Uri.EscapeDataString(raw_data);
string getURL_MoreFriendsAjax = "https://www.facebook.com/ajax/pagelet/generic.php/AllFriendsAppCollectionPagelet?data=" + encoded_raw_data + "&__user="******"&__a=1&__dyn=7n8ahyj2qmudwNAEU&__req=2";
string res_getURL_MoreFriendsAjax = chilkatHttpHelper.GetHtml(getURL_MoreFriendsAjax);
pgSource_Friends = res_getURL_MoreFriendsAjax;
string[] arry_UserData = Regex.Split(pgSource_Friends, "user.php");
ParseFriendIDs(arry_UserData, finalList_Friends);
tempCount_AjaxRequests++;
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error : " + ex.StackTrace);
}
} while ((collection_token != "" && cursor != "") || tempCount_AjaxRequests < 15 || pgSource_Friends.Contains(patternAllFriendsLoaded));
finalList_Friends.ForEach(delegate(String friendID)
{
if (friendID.Contains("&"))
{
friendID = friendID.Remove(friendID.IndexOf("&"));
}
list_finalFriendsID.Add(friendID);
});
list_finalFriendsID = list_finalFriendsID.Distinct().ToList();
}
catch (Exception ex)
{
GlobusLogHelper.log.Error("Error : " + ex.StackTrace);
}
return(list_finalFriendsID);
}
public static void RequestJSCSSIMG(string pageSource, ref Chilkat.Http http)
{
ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
List<string> listURLs = new List<string>();
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
//CSS Request
foreach (string item in GetHrefsFromString(pageSource))
{
if (item.Contains(".css"))
{
string cssSource = item.Replace(" ", "").Trim();
try
{
//string res = HttpHelper.GetHtmlProxy(cssSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
listURLs.Add(cssSource);
}
catch (Exception)
{
Thread.Sleep(500);
try
{
string res = HttpHelper.GetHtmlProxy(cssSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
}
catch (Exception)
{
}
}
}
}
//JS Request
string[] scriptArr = Regex.Split(pageSource, "/script>");
foreach (string item in scriptArr)
{
try
{
if (item.Contains("static.ak.") || item.Contains("profile.ak."))
{
int startIndx = item.LastIndexOf("src=") + "src=".Length + 1;
int endIndx = item.IndexOf(">", startIndx) - 1;
string jsSource = item.Substring(startIndx, endIndx - startIndx);
//if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak."))
if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.") || jsSource.StartsWith("http://profile.ak.") || jsSource.StartsWith("https://profile.ak.") || jsSource.StartsWith("http://s-profile.ak.") || jsSource.StartsWith("https://s-profile.ak."))
{
try
{
//string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
listURLs.Add(jsSource);
}
catch (Exception)
{
Thread.Sleep(500);
try
{
string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
}
catch (Exception)
{
}
}
}
}
}
catch (Exception)
{
}
}
string[] moreScriptArray = Regex.Split(pageSource, "\"src\":");
foreach (string item in moreScriptArray)
{
try
{
int startIndx = 1;
int endIndx = item.IndexOf("\"", startIndx);
string jsSource = item.Substring(startIndx, endIndx - startIndx).Replace("\\", "");
if (jsSource.Contains(".js"))
{
//string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
listURLs.Add(jsSource);
}
}
catch (Exception)
{
}
}
///IMG Request
string[] imageArr = Regex.Split(pageSource, "<img");
foreach (string item in imageArr)
{
try
{
if (item.Contains("static.ak.") || item.Contains("profile.ak."))
{
int startIndx = item.IndexOf("src=") + "src=".Length + 1;
int endIndx = item.IndexOf("\"", startIndx + 1);
string jsSource = item.Substring(startIndx, endIndx - startIndx);
//if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak."))
if (jsSource.StartsWith("http://static.ak.") || jsSource.StartsWith("https://static.ak.") || jsSource.StartsWith("http://s-static.ak.") || jsSource.StartsWith("https://s-static.ak.") || jsSource.StartsWith("http://profile.ak.") || jsSource.StartsWith("https://profile.ak.") || jsSource.StartsWith("http://s-profile.ak.") || jsSource.StartsWith("https://s-profile.ak."))
{
if (jsSource.Contains(".png") || jsSource.Contains(".gif") || jsSource.Contains(".jpg") || jsSource.Contains(".jpeg"))
{
try
{
//string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
listURLs.Add(jsSource);
}
catch (Exception)
{
Thread.Sleep(500);
try
{
string res = HttpHelper.GetHtmlProxy(jsSource, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);// HttpHelper.getHtmlfromUrl(new Uri(cssSource));
}
catch (Exception)
{
}
}
}
}
}
}
catch { };
}
listURLs = listURLs.Distinct().ToList();
foreach (string item in listURLs)
{
try
{
string res = HttpHelper.GetHtmlProxy(item, proxyAddress, proxyPort, proxyUsername, proxyPassword, ref http);
}
catch { };
}
}
public void StartAcceptInvitations(ref GlobusHttpHelper httpHelper)
{
try
{
string csrfToken = string.Empty;
string userFirstName = string.Empty;
string UserLastName = string.Empty;
string SenderName = string.Empty;
string newPagesource = string.Empty;
bool isTrue = false;
int startRow = 1;
string pageSource = httpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/inbox/invitations/pending"));
var resultForUserDetails = FindTheUserName(pageSource);
try
{
resultForUserDetails = resultForUserDetails.Substring(resultForUserDetails.IndexOf("alt="), resultForUserDetails.IndexOf("height") - resultForUserDetails.IndexOf("alt=")).Replace("alt=", string.Empty).Replace("/", string.Empty).Trim();
userFirstName = resultForUserDetails.Split(' ')[0].Replace("\"", string.Empty);
UserLastName = resultForUserDetails.Split(' ')[1].Replace("\"", string.Empty);
}
catch { }
if (pageSource.Contains("csrfToken"))
{
csrfToken = pageSource.Substring(pageSource.IndexOf("csrfToken"), 100);
string[] Arr = csrfToken.Split('"');
try
{
foreach (string item in Arr)
{
try
{
if (item.Contains("csrfToken="))
{
csrfToken = item.Substring(item.IndexOf("csrfToken="), item.IndexOf("&", item.IndexOf("csrfToken=")) - item.IndexOf("csrfToken=")).Replace("csrfToken=", string.Empty).Replace("\"", string.Empty).Replace("\\", string.Empty).Trim();//Arr[2].Replace(@"\", string.Empty).Replace("//", string.Empty);
break;
}
}
catch (Exception ex)
{
}
if (item.Contains("csrfToken="))
{
csrfToken = item.Replace("csrfToken=", string.Empty).Trim();
break;
}
}
}
catch (Exception ex)
{
//csrfToken = Arr[0].Replace("csrfToken=", "").Replace("\\", "");
}
}
// For Show More
//string postData1 = "pkey=inbox-invitations-pending&tcode=%5Bobject%20Arguments%5D&plist=";
//string response1 = httpHelper.postFormData(new Uri("http://www.linkedin.com/lite/web-action-track?csrfToken="+csrfToken+""),postData1);
//string pageSource2=httpHelper.getHtmlfromUrl(new Uri("http://www.linkedin.com/inbox/invitations/pending/more?sinceDate=1366351490125&startRow=6&count=20&showBlocked=false&ctx=inbox&rnd=1366353236172"));
//*** Conver HTML to XML *******************************//
#region Convert HTML to XML
ChilkatHttpHelpr objhelper = new ChilkatHttpHelpr();
//xHtml contain xml data
string xHtml = objhelper.ConvertHtmlToXml(pageSource);
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
//xHtml.
//// Iterate over all h1 tags:
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
#region Invitatin count
xBeginSearchAfter = null;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "invitation-count count ");
try
{
while ((xNode != null))
{
string strvalue = xNode.AccumulateTagContent("text", "script|style");
string Invitatincount = strvalue;
Log("[ " + DateTime.Now + " ] => [ Invitation Count = " + Invitatincount + " UserName = "******" ]");
Log("-----------------------------------------------------------------------------------------------------------------------------------");
break;
}
}
catch (Exception ex)
{
}
#endregion
do
{
newPagesource = httpHelper.getHtmlfromUrl1(new Uri("https://www.linkedin.com/inbox/invitations?keywords=&sortBy=&startRow=" + startRow + "&subFilter=&trk=&showBlocked=false"));
if (newPagesource.Contains("inbox-list"))
{
string inbox_list = httpHelper.GetDataWithTagValueByTagAndAttributeNameWithClass(newPagesource, "ol", "inbox-list ");
if (inbox_list.Contains("<li"))
{
isTrue = true;
string[] srrLi = Regex.Split(inbox_list, "<li");
foreach (string item in srrLi)
{
try
{
if (item.Contains("data-gid=\""))
{
string data_gid = item.Substring(item.IndexOf("data-gid=\"") + 10, item.IndexOf("\"", item.IndexOf("data-gid=\"") + 10) - (item.IndexOf("data-gid=\"") + 10)).Replace("\"", string.Empty).Replace("data-gid=\"", string.Empty).Trim();
int startindex1 = item.IndexOf("alt=");
string start1 = item.Substring(startindex1).Replace("alt=",string.Empty);
int endindex1 = start1.IndexOf("height");
string end1 = start1.Substring(0, endindex1).Replace("\"", string.Empty).Trim();
SenderName = end1;
string response2 = httpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/inbox/action?mboxItemGID=" + data_gid + "&actionType=invitationAccept&csrfToken=" + csrfToken + "&goback=%2Epiv_*1_*1_*1_*1_*1&trk=inbox-invitations-inv-accept&ctx=inbox&rnd=1366352095313"));
if (response2.Contains(" are now connected"))
{
string SuccessMsg = string.Empty;
int startindex = response2.IndexOf("<div class=\"confirmation\">");
if (startindex > 0)
{
try
{
string start = response2.Substring(startindex).Replace("<div class=\"confirmation\">", string.Empty);
int endindex = start.IndexOf("<ul>");
string end = start.Substring(0, endindex);
//SuccessMsg = end.Replace("<h4>", string.Empty).Replace("\"", string.Empty).Replace("</h4>", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("\"u002", "-").Replace("You", "User: "******"<h4>", string.Empty).Replace("\"", string.Empty).Replace("</h4>", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("\"u002", "-").Replace("You", "User: "******"( Name:" + userFirstName + " " + UserLastName + ") ").Trim();
}
catch { }
}
GlobusFileHelper.AppendStringToTextfileNewLine(SuccessMsg, Globals.path_AcceptInvitationEmail);
Log("[ " + DateTime.Now + " ] => [ " + SuccessMsg + " ]");
}
if (!(response2.Contains(SenderName)))
{
Log("[ " + DateTime.Now + " ] => [ Invitation accepted from " + SenderName + " ]");
}
else
{
//Log("There is some error !");
}
}
}
catch (Exception ex)
{
}
}
}
else
{
//Log("[ " + DateTime.Now + " ] => [ There is no invitation ! ]");
Log("[ " + DateTime.Now + " ] => [ No more invitations left to accept ! ]");
}
}
startRow = startRow + 10;
} while (newPagesource.Contains("is now a connection."));
//else
//{
// Log("[ " + DateTime.Now + " ] => [ There is no invitation ! ]");
//}
//if (isTrue)
//{
// StartAcceptInvitations(ref httpHelper);
//}
}
catch (Exception ex)
{
}
}
public string CreateBoard_new(string BoardName, string Category, ref PinInterestUser objPinUser)
{
try
{
string Checking = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri("https://www.pinterest.com/settings/"));
string ScreenName = objPinUser.ScreenName; //ObjAccountManager.Getscreen_NameRepin(ref objPinUser);
if (Checking.Contains("profileName"))
{
}
else
{
ObjAccountManager.LoginPinterestAccount(ref objPinUser);
}
string redirectDomain = GlobusHttpHelper.valueURl.Split('.')[0];
string newHomePageUrl = redirectDomain + "." + "pinterest.com";
if (!string.IsNullOrEmpty(Checking))
{
ChilkatHttpHelpr objChilkatHttpHelpr = new ChilkatHttpHelpr();
//string newpostdata = "source_url=%2F" + ScreenName + "%2F&data=%7B%22options%22%3A%7B%22name%22%3A%22" + (BoardName.Replace(" ", "+")) + "%22%2C%22category%22%3A%22other%22%2C%22description%22%3A%22%22%2C%22privacy%22%3A%22public%22%2C%22layout%22%3A%22default%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=App()%3EUserProfilePage(resource%3DUserResource(username%3D" + ScreenName + "%2C+invite_code%3Dnull))%3EUserProfileContent(resource%3DUserResource(username%3D" + objPinUser.ScreenName + "%2C+invite_code%3Dnull))%3EUserBoards()%3EGrid(resource%3DProfileBoardsResource(username%3D" + ScreenName + "))%3EGridItems(resource%3DProfileBoardsResource(username%3D" + ScreenName + "))%3EBoardCreateRep(ga_category%3Dboard_create%2C+text%3DCreate+a+board%2C+submodule%3D%5Bobject+Object%5D)%23Modal(module%3DBoardCreate())";
string newpostdata = "source_url=%2F" + ScreenName + "%2F&data=%7B%22options%22%3A%7B%22name%22%3A%22" + (BoardName.Replace(" ", "+")) + "%22%2C%22category%22%3A%22other%22%2C%22description%22%3A%22%22%2C%22privacy%22%3A%22public%22%2C%22layout%22%3A%22default%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=App%3EUserProfilePage%3EUserProfileContent%3EUserBoards%3EGrid%3EGridItems%3EBoardCreateRep(ga_category%3Dboard_create%2C+text%3DCreate+a+board%2C+submodule%3D%5Bobject+Object%5D)%23App%3EModalManager%3EModal(showCloseModal%3Dtrue%2C+mouseDownInModal%3Dfalse)";
string PostUrlBoard = redirectDomain + ".pinterest.com/resource/BoardResource/create/";
try
{
CreatedBoardPageSource = objPinUser.globusHttpHelper.postFormDataProxyPin(new Uri(PostUrlBoard), newpostdata, newHomePageUrl);
}
catch (Exception ex)
{
GlobusLogHelper.log.Error(" Error :" + ex.StackTrace);
}
if (CreatedBoardPageSource.Contains("error\": null"))
{
if (newpostdata.Contains("You already have a board with that name."))
{
GlobusLogHelper.log.Info(" => [ You already have a board with this name. " + BoardName + " For " + objPinUser.Username + " ]");
return(null);
}
try
{
string ModuleName = "AddBoardName";
string Status = "Board_Created";
QueryManager qm = new QueryManager();
qm.insertAccRePort(objPinUser.Username, ModuleName, "", BoardName, "", "", "", "", Status, "", "", DateTime.Now);
//qm.insertBoard_AddBoardName(objPinUser.Username, ModuleName, BoardName, Status);
objDelegateAccountReport();
}
catch (Exception ex)
{
GlobusLogHelper.log.Info(" => [ Board Created " + BoardName + " ]");
return(null);
}
GlobusLogHelper.log.Info(" => [ Successfully Created Board " + BoardName + " For " + objPinUser.Username + " ]");
string BoardId = objAddNewPinManager.GetBoardId_Board(BoardName, ref objPinUser);//GetBoardId(BoardName, ref pinterestAccountManager);
return(BoardId);
}
else if (CreatedBoardPageSource.Contains("You have a board with this name."))
{
GlobusLogHelper.log.Info(" => [ You already have a board with this name. " + BoardName + " For " + objPinUser.Username + " ]");
string BoardId = objAddNewPinManager.GetBoardId_Board(BoardName, ref objPinUser);
return(BoardId);
}
else
{
GlobusLogHelper.log.Info(" => [ Board Creation Process Failed " + BoardName + " ]");
return(CreatedBoardPageSource);
}
}
else
{
GlobusLogHelper.log.Info(" => [ Login Issue " + " For " + objPinUser.Username + " ]");
}
}
catch (Exception ex)
{
GlobusLogHelper.log.Info(" => [ Board Creation Process Failed " + BoardName + " ]");
return(null);
}
return(null);
}
public List<StructTweetIDs> NewKeywordStructDataSearchByPeople(string keyword)
{
try
{
BaseLib.GlobusRegex regx = new GlobusRegex();
int counter = 0;
lst_structTweetIDs = new List<StructTweetIDs>();
string res_Get_searchURL = string.Empty;
string searchURL = string.Empty;
string maxid = string.Empty;
string TweetId = string.Empty;
string text = string.Empty;
string ProfileName = string.Empty;
string Location = string.Empty;
string Bio = string.Empty;
string website = string.Empty;
string NoOfTweets = string.Empty;
string Followers = string.Empty;
string Followings = string.Empty;
if (keyword.Trim().Contains(" "))
{
keyword = keyword.Replace(" ", "+");
}
startAgain:
if (!RetweetFromUserName)
{
if (counter == 0)
{
//searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + "";
//searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime";
//29-4-2014 only for client it has been changed
searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&mode=users";
counter++;
}
else
{
//searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + "";
//29-4-2014 only for client it is changed
searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&mode=users&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + "";
}
}
else
{
searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + RetweetExtractcount + "&include_available_features=1&include_entities=1";
}
try
{
res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
if (string.IsNullOrEmpty(res_Get_searchURL))
{
res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
}
try
{
//string sjss = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor");
//splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
if (item.Contains("refresh_cursor"))
{
int startIndex = item.IndexOf("USER-");
string start = item.Substring(startIndex).Replace("data-user-id=\\\"", "");
int endIndex = start.IndexOf("\"");
string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "");
TweetId = end;
}
if (item.Contains("scroll_cursor"))
{
int startIndex = item.IndexOf("USER-");
string start = item.Substring(startIndex).Replace("data-user-id=\\\"", "");
int endIndex = start.IndexOf("\"");
string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "");
TweetId = end;
}
}
}
catch (Exception)
{
}
}
catch (Exception ex)
{
System.Threading.Thread.Sleep(2000);
res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
// && !res_Get_searchURL.Contains("has_more_items\":false")
if (!string.IsNullOrEmpty(res_Get_searchURL))
{
//string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); //Regex.Split(res_Get_searchURL, "\"in_reply_to_status_id_str\"");
string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id");
splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
if (item.Contains("data-screen-name=") && !item.Contains("js-actionable-user js-profile-popup-actionable"))
{
//var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL);
//string DataHtml = (string)avc["items_html"];
}
else
{
//continue;
}
string modified_Item = "\"from_user\"" + item;
string id = "";
try
{
int startIndex = item.IndexOf("data-user-id=");
string start = item.Substring(startIndex).Replace("data-user-id=\\\"", "");
int endIndex = start.IndexOf("\\\"");
string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "");
id = end;
}
catch (Exception ex)
{
id = "null";
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
string from_user_id = "";
try
{
int startIndex = item.IndexOf("data-screen-name=\\\"");
string start = item.Substring(startIndex).Replace("data-screen-name=\\\"", "");
int endIndex = start.IndexOf("\\\"");
string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", "");
from_user_id = end;
}
catch (Exception ex)
{
from_user_id = "null";
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
string tweetUserid = string.Empty;
try
{
int startIndex = item.IndexOf("=\\\"");
string start = item.Substring(startIndex).Replace("=\\\"", "");
int endIndex = start.IndexOf("\\\"");
string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", "");
tweetUserid = end;
}
catch (Exception ex)
{
from_user_id = "null";
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
StructTweetIDs structTweetIDs = new StructTweetIDs();
if (id != "null")
{
structTweetIDs.ID_Tweet = tweetUserid;
structTweetIDs.ID_Tweet_User = id;
structTweetIDs.username__Tweet_User = from_user_id;
structTweetIDs.wholeTweetMessage = text;
lst_structTweetIDs.Add(structTweetIDs);
Log("[ " + DateTime.Now + " ] => [ " + tweetUserid + " ]");
Log("[ " + DateTime.Now + " ] => [ " + id + " ]");
Log("[ " + DateTime.Now + " ] => [ " + from_user_id + " ]");
Log("-------------------------------------------------------------------------------------------------------------------------------");
if (!File.Exists(Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"))
{
GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS", Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv");
}
//foreach (TwitterDataScrapper.StructTweetIDs item in data)
{
ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr();
GlobusHttpHelper HttpHelper = new GlobusHttpHelper();
string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + from_user_id), "", "");
string Responce = ProfilePageSource;
#region Convert HTML to XML
string xHtml = objChilkat.ConvertHtmlToXml(Responce);
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
int counterdata = 0;
xBeginSearchAfter = null;
string dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterdata == 0)
{
ProfileName = xNode.AccumulateTagContent("text", "script|style");
counterdata++;
}
else if (counterdata == 1)
{
website = xNode.AccumulateTagContent("text", "script|style");
counterdata++;
}
else
{
break;
}
// xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor");
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim();
break;
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Location = xNode.AccumulateTagContent("text", "script|style");
break;
}
int counterData = 0;
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterData == 0)
{
// NoOfTweets = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileNav-value");
NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty);
counterData++;
}
else if (counterData == 1)
{
Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty);
counterData++;
}
else if (counterData == 2)
{
Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty);
counterData++;
}
else
{
break;
}
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
}
if (!string.IsNullOrEmpty(from_user_id) && tweetUserid != "null")
{
string Id_user = tweetUserid.Replace("}]", string.Empty).Trim();
Globals.lstScrapedUserIDs.Add(Id_user);
GlobusFileHelper.AppendStringToTextfileNewLine(id + "," + from_user_id + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv");
GlobusFileHelper.AppendStringToTextfileNewLine(from_user_id, Globals.Path_KeywordScrapedListData + "-" + keyword + ".txt");
Log("[ " + DateTime.Now + " ] => [ " + from_user_id + "," + Id_user + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]");
}
}
}
lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList();
if (lst_structTweetIDs.Count >= noOfRecords)
{
return lst_structTweetIDs;
}
}
if (lst_structTweetIDs.Count <= noOfRecords)
{
maxid = lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet;
if (res_Get_searchURL.Contains("has_moreitems\":false"))
{
return lst_structTweetIDs;
}
else
{
goto startAgain;
}
}
else
{
if (res_Get_searchURL.Contains("has_more_items\":false"))
{
return lst_structTweetIDs;
}
else
goto startAgain;
}
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return lst_structTweetIDs;
}
public static string GetUserNameFromUserId(string userid)
{
string username = string.Empty;
ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
clsDBQueryManager DB = new clsDBQueryManager();
DataSet ds = DB.GetUserName(userid);
string user_id = string.Empty;
if (ds.Tables[0].Rows.Count > 0)
{
foreach (DataRow dataRow in ds.Tables["tb_UsernameDetails"].Rows)
{
user_id = dataRow.ItemArray[0].ToString();
return user_id;
}
}
try
{
string PageSource = string.Empty;
if (!string.IsNullOrEmpty(userid) && NumberHelper.ValidateNumber(userid))
{
PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.xml?user_id=" + userid + "&include_entities=true");
if (!PageSource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour."))
{
try
{
int startIndex = PageSource.IndexOf("<screen_name>");
if (startIndex > 0)
{
string Start = PageSource.Substring(startIndex);
int endIndex = Start.IndexOf("</screen_name>");
string End = Start.Substring(0, endIndex);
username = End.Replace("<screen_name>", "");
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
else
{
username = "******";
}
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetUserNameFromUserId() -- " + userid + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return username;
}
//public List<string> GetRetweetData_Scrape(string keyword, out string returnStatus)
//{
// string status = string.Empty;
// List<string> lst_ReTweetIDs = new List<string>();
// try
// {
// string searchURL = string.Empty;
// if (!NumberHelper.ValidateNumber(keyword))
// {
// searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + RetweetExtractcount + "&include_available_features=1&include_entities=1";
// //searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?screen_name=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true";
// //searchURL = "https://api.twitter.com/1.1/statuses/retweeted_by_user.json?screen_name=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true";
// }
// else if (NumberHelper.ValidateNumber(keyword))
// {
// searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?id=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true";
// }
// //string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
// ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
// string res_Get_searchURL = HttpHelper.GetHtml(searchURL);
// // if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && res_Get_searchURL.Contains("created_at") && !string.IsNullOrEmpty(res_Get_searchURL))
// if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && !string.IsNullOrEmpty(res_Get_searchURL))
// {
// string[] arrGetRetweeted = Regex.Split(res_Get_searchURL, "js-tweet-text tweet-text");
// foreach (string item in arrGetRetweeted)
// {
// try
// {
// if (item.Contains("Retweeted by"))
// {
// int Startindex = item.IndexOf("\\\"");
// string start = item.Substring(Startindex);
// int EndIndex = start.IndexOf("\\/p");
// string End = start.Substring(0, EndIndex).Replace("\\\"", "").Replace("\\u003", "").Replace("&", "");
// End = End.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "").Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); ;
// }
// }
// // catch { }
// //}
// //string[] splitRes = Regex.Split(res_Get_searchURL, "status");//Regex.Split(res_Get_searchURL, "{\"created_at\"");
// //splitRes = splitRes.Skip(1).ToArray();
// //foreach (string item in splitRes)
// //{
// // string Tweet = string.Empty;
// // string Tweeter = string.Empty;
// // try
// // {
// // int startIndex = item.IndexOf("<text>");
// // string start = item.Substring(startIndex);
// // int endIndex = start.IndexOf("</text>");
// // string end = start.Substring(0, endIndex);
// // Tweet = end.Replace("<text>", "");
// // int startOfInndex = Tweet.IndexOf(":");
// // Tweeter = Tweet.Substring(0, startOfInndex);
// // Tweet = Tweet.Replace(Tweeter, "");
// // }
// catch (Exception ex)
// {
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
// }
// //string data = keyword + ":" + Tweeter + ":" + Tweet.Replace(":", "^");
// //Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(data, Globals.Path_RETweetExtractor);
// //lst_ReTweetIDs.Add(data);
// }
// }
// else if (res_Get_searchURL.Contains("Rate limit exceeded"))
// {
// status = "Rate limit exceeded";
// }
// else if (res_Get_searchURL.Contains("Sorry, that page does not exist"))
// {
// status = "Sorry, that page does not exist";
// }
// else if (res_Get_searchURL.Contains("Not authorized"))
// {
// status = "Not Authorized";
// }
// else if (string.IsNullOrEmpty(res_Get_searchURL))
// {
// status = "Not Authorized";
// }
// else
// {
// status = "Empty";
// }
// }
// catch (Exception ex)
// {
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
// status = "Error";
// }
// returnStatus = status;
// return lst_ReTweetIDs;
//}
#endregion
public List<string> GetRetweetData_Scrape(string keyword, out string returnStatus)
{
string status = string.Empty;
List<string> lst_ReTweetIDs = new List<string>();
try
{
string searchURL = string.Empty;
int extendendCount = 0;
extendendCount = RetweetExtractcount * 10;
if (!NumberHelper.ValidateNumber(keyword))
{
searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + extendendCount + "&include_available_features=1&include_entities=1";
}
else if (NumberHelper.ValidateNumber(keyword))
{
searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?id=" + keyword + "&count=" + extendendCount + "&include_entities=true";
}
//string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr();
string res_Get_searchURL = HttpHelper.GetHtml(searchURL);
if (string.IsNullOrEmpty(res_Get_searchURL))
{
res_Get_searchURL = HttpHelper.GetHtml(searchURL);
}
var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL);
string DataHtml = string.Empty;
try
{
DataHtml = (string)avc["items_html"];
}
catch { };
if (string.IsNullOrEmpty(DataHtml))
{
foreach (object data in avc)
{
DataHtml = data.ToString();
}
}
string[] splitRes = Regex.Split(DataHtml, "ProfileTweet u-textBreak js-tweet js-stream-tweet js-actionable-tweet");//Regex.Split(res_Get_searchURL, "{\"created_at\"");
if (splitRes.Count() == 1)
{
splitRes = Regex.Split(DataHtml, "ProfileTweet-action ProfileTweet-action--more js-more-ProfileTweet-actions");
}
splitRes = splitRes.Skip(1).ToArray();
GlobusRegex regx = new GlobusRegex();
foreach (string item in splitRes)
{
string text = string.Empty;
string tweetURL = string.Empty;
string tweetUserid = string.Empty;
if (item.Contains("Icon--retweeted"))//retweeted Icon
{
try
{
//int startindex = item.IndexOf("ProfileTweet-text js-tweet-text u-dir");
//string start = item.Substring(startindex).Replace("ProfileTweet-text js-tweet-text u-dir", "").Replace("js-tweet-text tweet-text tweet-text-rtl\\\"", "");
//int endindex = start.IndexOf("</p>");
//if (endindex == -1)
//{
// endindex = 0;
// endindex = start.IndexOf("stream-item-footer");
//}
//string end = start.Substring(0, endindex);
//end = regx.StripTagsRegex(end);
#region sonu edited code
try
{
string[] getText = Regex.Split(item, "TweetTextSize TweetTextSize--16px js-tweet-text tweet-text");
try
{
text = Utils.getBetween(getText[1], ">", "<");
}
catch { };
}catch{};
#endregion
//text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "");
text = text.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "").Replace(" "," ");
text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("'", "'").Replace("&", "&").Replace("=>", "=>").Replace(" ", " ").Replace("\\", "").Replace("title=", "").Replace("js-tweet-text tweet-text", "").Replace(" ", " ");
string[] array = Regex.Split(text, "http");
text = string.Empty;
foreach (string itemData in array)
{
if (!itemData.Contains("t.co"))
{
string data = string.Empty;
if (itemData.Contains("//"))
{
data = ("http" + itemData).Replace(" span ", string.Empty);
if (!text.Contains(itemData.Replace(" ", "")))
{
text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("lang=en", string.Empty).Replace("dir=ltr data-aria-label-part=0", string.Empty);
}
}
else
{
if (!text.Contains(itemData.Replace(" ", "")))
{
text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("lang=en", string.Empty).Replace("dir=ltr data-aria-label-part=0", string.Empty).Replace("\n",string.Empty).Replace(" ",string.Empty);
}
}
}
}
if (text.Contains("data-aria-label-part=0"))
{
text = globushttpHelper.getBetween(text + ":&$#@", "data-aria-label-part=0", ":&$#@").Replace(" "," ");
}
//data-permalink-path="
#region Scrap
try
{
//data-permalink-path="
int startIndex = item.IndexOf("data-tweet-id=\"");
string start1 = item.Substring(startIndex).Replace("data-tweet-id=\"", "");
int endIndex = start1.IndexOf("\"");
string end1 = start1.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", "");
tweetUserid = end1;
}
catch { };
#endregion
#region Scrap Url
try
{
//data-permalink-path="
int startIndex = item.IndexOf("data-permalink-path=\"");
string start1 = item.Substring(startIndex).Replace("data-permalink-path=\"", "");
int endIndex = start1.IndexOf("\"");
string end1 = start1.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", "");
tweetURL = "https://twitter.com" + end1;
}
catch { };
#endregion
//Remove RT or @ mentions
#region -----RT or @ mentions-------
if (RemoveRTMSg && text.StartsWith("RT"))
continue;
if (removeAtMentions && text.Contains("@"))
continue;
#endregion
if (lst_ReTweetIDs.Count() < RetweetExtractcount)
{
string txtdata = keyword + ":" + (text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("#", string.Empty)).Replace("dir=ltr", "").Replace(" ", " ");
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(txtdata, Globals.Path_RETweetExtractor);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("#", string.Empty).Replace("dir=ltr", "").Trim(), Globals.Path_RETweetExtractorUpload);
lst_ReTweetIDs.Add(txtdata);
//generate CSV file
if (!File.Exists(Globals.Path_RETweetExtractorCSV))
{
string Header = "UserId" + "," + "Keyword" + "," + "Message" + "," + "URL";
GlobusFileHelper.AppendStringToTextfileNewLine(Header, Globals.Path_RETweetExtractorCSV);
}
string CheckReTweetExtractor = tweetUserid + "," + keyword + "," + text.Replace(","," ") + "," + tweetURL;
if (!string.IsNullOrEmpty(tweetURL))
{
GlobusFileHelper.AppendStringToTextfileNewLine(CheckReTweetExtractor, Globals.Path_RETweetExtractorCSV);
//AddToLog_Checker("[ " + DateTime.Now + " ] => [ " + userEmail + "-" + userID + "-" + ScreanName + " ]");
}
}
status = "No Error";
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
}
}
catch (Exception ex)
{
status = "Empty";
}
returnStatus = status;
return lst_ReTweetIDs;
}
public void StartAcceptInvitations(ref GlobusHttpHelper httpHelper)
{
try
{
string csrfToken = string.Empty;
string userFirstName = string.Empty;
string UserLastName = string.Empty;
string SenderName = string.Empty;
string newPagesource = string.Empty;
bool isTrue = false;
int startRow = 1;
string pageSource = httpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/inbox/invitations/pending"));
var resultForUserDetails = FindTheUserName(pageSource);
try
{
resultForUserDetails = resultForUserDetails.Substring(resultForUserDetails.IndexOf("alt="), resultForUserDetails.IndexOf("height") - resultForUserDetails.IndexOf("alt=")).Replace("alt=", string.Empty).Replace("/", string.Empty).Trim();
userFirstName = resultForUserDetails.Split(' ')[0].Replace("\"", string.Empty);
UserLastName = resultForUserDetails.Split(' ')[1].Replace("\"", string.Empty);
}
catch { }
if (pageSource.Contains("csrfToken"))
{
csrfToken = pageSource.Substring(pageSource.IndexOf("csrfToken"), 100);
string[] Arr = csrfToken.Split('"');
try
{
foreach (string item in Arr)
{
try
{
if (item.Contains("csrfToken="))
{
csrfToken = item.Substring(item.IndexOf("csrfToken="), item.IndexOf("&", item.IndexOf("csrfToken=")) - item.IndexOf("csrfToken=")).Replace("csrfToken=", string.Empty).Replace("\"", string.Empty).Replace("\\", string.Empty).Trim();//Arr[2].Replace(@"\", string.Empty).Replace("//", string.Empty);
break;
}
}
catch (Exception ex)
{
}
if (item.Contains("csrfToken="))
{
csrfToken = item.Replace("csrfToken=", string.Empty).Trim();
break;
}
}
}
catch (Exception ex)
{
//csrfToken = Arr[0].Replace("csrfToken=", "").Replace("\\", "");
}
}
// For Show More
//string postData1 = "pkey=inbox-invitations-pending&tcode=%5Bobject%20Arguments%5D&plist=";
//string response1 = httpHelper.postFormData(new Uri("http://www.linkedin.com/lite/web-action-track?csrfToken="+csrfToken+""),postData1);
//string pageSource2=httpHelper.getHtmlfromUrl(new Uri("http://www.linkedin.com/inbox/invitations/pending/more?sinceDate=1366351490125&startRow=6&count=20&showBlocked=false&ctx=inbox&rnd=1366353236172"));
//*** Conver HTML to XML *******************************//
#region Convert HTML to XML
ChilkatHttpHelpr objhelper = new ChilkatHttpHelpr();
//xHtml contain xml data
string xHtml = objhelper.ConvertHtmlToXml(pageSource);
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
//xHtml.
//// Iterate over all h1 tags:
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
#region Invitatin count
xBeginSearchAfter = null;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "invitation-count count ");
try
{
while ((xNode != null))
{
string strvalue = xNode.AccumulateTagContent("text", "script|style");
string Invitatincount = strvalue;
Log("[ " + DateTime.Now + " ] => [ Invitation Count = " + Invitatincount + " UserName = "******" ]");
Log("-----------------------------------------------------------------------------------------------------------------------------------");
break;
}
}
catch (Exception ex)
{
}
#endregion
do
{
newPagesource = httpHelper.getHtmlfromUrl1(new Uri("https://www.linkedin.com/inbox/invitations?keywords=&sortBy=&startRow=" + startRow + "&subFilter=&trk=&showBlocked=false"));
if (newPagesource.Contains("inbox-list"))
{
string inbox_list = httpHelper.GetDataWithTagValueByTagAndAttributeNameWithClass(newPagesource, "ol", "inbox-list ");
if (inbox_list.Contains("<li"))
{
isTrue = true;
string[] srrLi = Regex.Split(inbox_list, "<li");
foreach (string item in srrLi)
{
try
{
if (item.Contains("data-gid=\""))
{
string data_gid = item.Substring(item.IndexOf("data-gid=\"") + 10, item.IndexOf("\"", item.IndexOf("data-gid=\"") + 10) - (item.IndexOf("data-gid=\"") + 10)).Replace("\"", string.Empty).Replace("data-gid=\"", string.Empty).Trim();
int startindex1 = item.IndexOf("alt=");
string start1 = item.Substring(startindex1).Replace("alt=", string.Empty);
int endindex1 = start1.IndexOf("height");
string end1 = start1.Substring(0, endindex1).Replace("\"", string.Empty).Trim();
SenderName = end1;
string response2 = httpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/inbox/action?mboxItemGID=" + data_gid + "&actionType=invitationAccept&csrfToken=" + csrfToken + "&goback=%2Epiv_*1_*1_*1_*1_*1&trk=inbox-invitations-inv-accept&ctx=inbox&rnd=1366352095313"));
if (response2.Contains(" are now connected"))
{
string SuccessMsg = string.Empty;
int startindex = response2.IndexOf("<div class=\"confirmation\">");
if (startindex > 0)
{
try
{
string start = response2.Substring(startindex).Replace("<div class=\"confirmation\">", string.Empty);
int endindex = start.IndexOf("<ul>");
string end = start.Substring(0, endindex);
//SuccessMsg = end.Replace("<h4>", string.Empty).Replace("\"", string.Empty).Replace("</h4>", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("\"u002", "-").Replace("You", "User: "******"<h4>", string.Empty).Replace("\"", string.Empty).Replace("</h4>", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("\"u002", "-").Replace("You", "User: "******"( Name:" + userFirstName + " " + UserLastName + ") ").Trim();
}
catch { }
}
GlobusFileHelper.AppendStringToTextfileNewLine(SuccessMsg, Globals.path_AcceptInvitationEmail);
Log("[ " + DateTime.Now + " ] => [ " + SuccessMsg + " ]");
}
if (!(response2.Contains(SenderName)))
{
Log("[ " + DateTime.Now + " ] => [ Invitation accepted from " + SenderName + " ]");
}
else
{
//Log("There is some error !");
}
}
}
catch (Exception ex)
{
}
}
}
else
{
//Log("[ " + DateTime.Now + " ] => [ There is no invitation ! ]");
Log("[ " + DateTime.Now + " ] => [ No more invitations left to accept ! ]");
}
}
startRow = startRow + 10;
} while (newPagesource.Contains("is now a connection."));
//else
//{
// Log("[ " + DateTime.Now + " ] => [ There is no invitation ! ]");
//}
//if (isTrue)
//{
// StartAcceptInvitations(ref httpHelper);
//}
}
catch (Exception ex)
{
}
}
public void scrapUserInfo(object param)
{
try
{
Array paramsArray = new object[1];
paramsArray = (Array)param;
string UserName = (string)paramsArray.GetValue(0);
string userId = string.Empty;
string ProfileName = string.Empty;
string Location = string.Empty;
string Bio = string.Empty;
string website = string.Empty;
string NoOfTweets = string.Empty;
string Followers = string.Empty;
string Followings = string.Empty;
string IsProfilePIc = string.Empty;
ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr();
GlobusHttpHelper HttpHelper = new GlobusHttpHelper();
string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + UserName.Trim()), "", "");
if (string.IsNullOrEmpty(ProfilePageSource))
{
ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + UserName.Trim()), "", "");
}
if (string.IsNullOrEmpty(ProfilePageSource))
{
AddToLog_ScrapMember("[ " + DateTime.Now + " ] => [ User " + UserName + " is not exist or page source getting null.]");
return;
}
if (ProfilePageSource.Contains("Account suspended"))
{
AddToLog_ScrapMember("[ " + DateTime.Now + " ] => [ User " + UserName + " is suspended ]");
return;
}
string Responce = ProfilePageSource;
#region Convert HTML to XML
string xHtml = objChilkat.ConvertHtmlToXml(Responce);
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
int counterdata = 0;
xBeginSearchAfter = null;
string dataDescription = string.Empty;
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterdata == 0)
{
ProfileName = xNode.AccumulateTagContent("text", "script|style");
if (ProfileName.Contains("Verified account"))
{
ProfileName = ProfileName.Replace("Verified account", " ");
}
counterdata++;
}
else if (counterdata == 1)
{
website = xNode.AccumulateTagContent("text", "script|style");
if (website.Contains("Twitter Status"))
{
website = "";
}
counterdata++;
}
else
{
break;
}
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor");
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim();
break;
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "location profile-field");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Location = xNode.AccumulateTagContent("text", "script|style");
break;
}
int counterData = 0;
xBeginSearchAfter = null;
dataDescription = string.Empty;
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "data-element-term", "tweet_stats");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterData == 0)
{
NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty);
counterData++;
}
else if (counterData == 1)
{
Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty);
counterData++;
}
else if (counterData == 2)
{
Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty);
counterData++;
}
else
{
break;
}
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
}
try
{
int startindex = ProfilePageSource.IndexOf("profile_id");
string start = ProfilePageSource.Substring(startindex).Replace("profile_id", "");
int endindex = start.IndexOf(",");
string end = start.Substring(0, endindex).Replace(""", "").Replace("\"", "").Replace(":", "").Trim();
userId = end.Trim();
if (userId.Length > 15)
{
startindex = ProfilePageSource.IndexOf("profile_id"");
start = ProfilePageSource.Substring(startindex).Replace("profile_id"", "");
endindex = start.IndexOf(",");
end = start.Substring(0, endindex).Replace(""", "").Replace("\"", "").Replace(":", "").Replace(";", "").Trim();
userId = end.Trim();
}
}
catch { }
if (ProfilePageSource.Contains("default_profile_6_400x400") || ProfilePageSource.Contains("default_profile_5_400x400") || ProfilePageSource.Contains("default_profile_4_400x400") || ProfilePageSource.Contains("default_profile_3_400x400") || ProfilePageSource.Contains("default_profile_2_400x400") || ProfilePageSource.Contains("default_profile_1_400x400") || ProfilePageSource.Contains("default_profile_0_400x400"))
{
IsProfilePIc = "No";
}
else
{
IsProfilePIc = "Yes";
}
if (!File.Exists(Globals.Path_UserListInfoData))
{
GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS, ProfilePic", Globals.Path_UserListInfoData);
}
if (!string.IsNullOrEmpty(UserName))
{
//string Id_user = item.ID_Tweet_User.Replace("}]", string.Empty).Trim();
//Globals.lstScrapedUserIDs.Add(Id_user);
GlobusFileHelper.AppendStringToTextfileNewLine(userId + "," + UserName + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", "") + "," + IsProfilePIc, Globals.Path_UserListInfoData);
AddToLog_ScrapMember("[ " + DateTime.Now + " ] => [ " + userId + "," + UserName + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ," + IsProfilePIc + "]");
}
}
catch { }
}