private List<string> GetReplyTime(Globussoft.GlobusHttpHelper globusHttpHelper, string pageSource)
{
List<string> lstReplyTime = new List<string>();
try
{
if (pageSource.Contains("tweets-wrapper"))
{
try
{
string ReplyUserName = globusHttpHelper.GetDataWithTagValueByTagAndAttributeName(pageSource, "div", "tweets-wrapper");
if (ReplyUserName.Contains("client-and-actions") || ReplyUserName.Contains("_timestamp js-short-timestamp "))
{
if (ReplyUserName.Contains("_timestamp js-short-timestamp "))
{
List<string> lstTime = globusHttpHelper.GetTextDataByTagAndAttributeName(ReplyUserName, "span", "_timestamp js-short-timestamp ");
foreach (string item in lstTime)
{
try
{
if (item.Contains("-"))
{
string[] arrItem = Regex.Split(item, "-");
if (arrItem.Count() > 1)
{
lstReplyTime.Add(arrItem[1].Trim());
}
}
else
{
lstReplyTime.Add(item.Trim());
}
}
catch
{
}
}
}
if (ReplyUserName.Contains("client-and-actions"))
{
List<string> lstTime = globusHttpHelper.GetTextDataByTagAndAttributeName(ReplyUserName, "div", "client-and-actions");
foreach (string item in lstTime)
{
try
{
if (item.Contains("-"))
{
string[] arrItem = Regex.Split(item, "-");
if (arrItem.Count() > 1)
{
lstReplyTime.Add(arrItem[1].Trim());
}
}
else
{
lstReplyTime.Add(item.Trim());
}
}
catch
{
}
}
}
}
}
catch
{
}
}
}
catch
{
}
return lstReplyTime;
}
private List<string> GetReplyUserName(Globussoft.GlobusHttpHelper globusHttpHelper, string pageSource)
{
List<string> lstReplyUserName = new List<string>();
try
{
//string pageSource = globusHttpHelper.getHtmlfromUrl(new Uri(url), "", "");
if (pageSource.Contains("tweets-wrapper"))
{
try
{
string ReplyUserName = globusHttpHelper.GetDataWithTagValueByTagAndAttributeName(pageSource, "div", "tweets-wrapper");
if (ReplyUserName.Contains("username js-action-profile-name"))
{
try
{
List<string> lst_ReplyUserName = globusHttpHelper.GetTextDataByTagAndAttributeName(ReplyUserName, "span", "username js-action-profile-name");
foreach (string item in lst_ReplyUserName)
{
try
{
if (!string.IsNullOrEmpty(item) && !string.IsNullOrWhiteSpace(item))
{
lstReplyUserName.Add(item);
}
}
catch
{
}
}
}
catch
{
}
}
}
catch
{
}
}
}
catch
{
}
lstReplyUserName = lstReplyUserName.Distinct().ToList();
return lstReplyUserName;
}
private List<string> GetTweetTime(Globussoft.GlobusHttpHelper globusHttpHelper, string pageSource)
{
List<string> lstTweetTime = new List<string>();
try
{
if (pageSource.Contains("permalink-inner permalink-tweet-container"))
{
// string[] arrTweetUsername = Regex.Split(pageSource, "permalink-inner permalink-tweet-container");
string TweetUserName = globusHttpHelper.GetDataWithTagValueByTagAndAttributeName(pageSource, "div", "permalink-inner permalink-tweet-container");
if (TweetUserName.Contains("client-and-actions") || TweetUserName.Contains("_timestamp js-short-timestamp "))
{
if (TweetUserName.Contains("_timestamp js-short-timestamp "))
{
List<string> lstTime = globusHttpHelper.GetTextDataByTagAndAttributeName(TweetUserName, "span", "_timestamp js-short-timestamp ");
foreach (string item in lstTime)
{
try
{
if (item.Contains("-"))
{
string[] arrItem = Regex.Split(item, "-");
if (arrItem.Count() > 1)
{
lstTweetTime.Add(arrItem[1].Trim());
}
}
else
{
lstTweetTime.Add(item.Trim());
}
}
catch
{
}
}
}
//if (lstTweetTime.Count < 1)
{
if (TweetUserName.Contains("client-and-actions"))
{
List<string> lstTime = globusHttpHelper.GetTextDataByTagAndAttributeName(TweetUserName, "div", "client-and-actions");
foreach (string item in lstTime)
{
try
{
if (item.Contains("-"))
{
string[] arrItem = Regex.Split(item, "-");
if (arrItem.Count() > 1)
{
lstTweetTime.Add(arrItem[1].Trim());
}
}
else
{
lstTweetTime.Add(item.Trim());
}
}
catch
{
}
}
}
}
}
}
}
catch
{
}
return lstTweetTime;
}
private List<string> GetReply(Globussoft.GlobusHttpHelper globusHttpHelper, string pageSource)
{
List<string> lstReply = new List<string>();
try
{
//string pageSource = globusHttpHelper.getHtmlfromUrl(new Uri(url), "", "");
if (pageSource.Contains("tweets-wrapper"))
{
try
{
string TweetUserId = globusHttpHelper.GetDataWithTagValueByTagAndAttributeName(pageSource, "div", "tweets-wrapper");
if (TweetUserId.Contains("js-tweet-text"))
{
List<string> lstTweetReply = globusHttpHelper.GetTextDataByTagAndAttributeName(TweetUserId, "p", "js-tweet-text");
foreach (string item in lstTweetReply)
{
lstReply.Add(item.Replace("&", string.Empty).Replace("quot;", string.Empty));
}
//lstReply.AddRange(lstTweetReply);
}
}
catch
{
}
}
}
catch
{
}
return lstReply;
}
private List<string> GetTweetUserName(Globussoft.GlobusHttpHelper globusHttpHelper, string pageSource)
{
List<string> lstTweetUserName = new List<string>();
try
{
//string pageSource = globusHttpHelper.getHtmlfromUrl(new Uri(url), "", "");
if (pageSource.Contains("permalink-inner permalink-tweet-container"))
{
// string[] arrTweetUsername = Regex.Split(pageSource, "permalink-inner permalink-tweet-container");
string TweetUserName = globusHttpHelper.GetDataWithTagValueByTagAndAttributeName(pageSource, "div", "permalink-inner permalink-tweet-container");
if (TweetUserName.Contains("username js-action-profile-name"))
{
List<string> lst_TweetUserName = globusHttpHelper.GetTextDataByTagAndAttributeName(TweetUserName, "span", "username js-action-profile-name");
lstTweetUserName.AddRange(lst_TweetUserName);
}
}
}
catch
{
}
lstTweetUserName = lstTweetUserName.Distinct().ToList();
return lstTweetUserName;
}
private List<string> GetTweet(Globussoft.GlobusHttpHelper globusHttpHelper, string pageSource)
{
List<string> lstTweet = new List<string>();
try
{
//string pageSource=globusHttpHelper.getHtmlfromUrl(new Uri(url),"","");
if (pageSource.Contains("permalink-inner permalink-tweet-container") && pageSource.Contains("js-tweet-text tweet-text "))
{
//string[] arrTweet = Regex.Split(pageSource, "js-tweet-text tweet-text");
List<string> lstTweetUserName = globusHttpHelper.GetTextDataByTagAndAttributeName(pageSource, "p", "js-tweet-text tweet-text ");
if (lstTweetUserName.Count > 0)
{
foreach (string item in lstTweetUserName)
{
try
{
string tweet = item.Replace(""", string.Empty).Replace("&", string.Empty);
lstTweet.Add(tweet);
}
catch
{
}
}
}
}
}
catch
{
}
return lstTweet;
}
private void GetMentions_ThroughAjax(ref Globussoft.GlobusHttpHelper globusHttpHelper, ref string userId, ref string userName, ref string Screen_name, ref string postAuthenticityToken, string MaxId)
{
string tweetId = string.Empty;
string tweetUserName = string.Empty;
string tweetMessage = string.Empty;
string max_id = MaxId;
try
{
string status = globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/mentions/timeline?include_available_features=1&include_entities=1&max_id=" + max_id), "", "");
if (status.Contains("div class=\\\"content"))
{
string[] arrContent = Regex.Split(status, @"div class=\\\""content");
foreach (string item in arrContent)
{
try
{
if (item.Contains("username js-action-profile-name"))
{
try
{
try
{
tweetUserName = item.Substring(item.IndexOf("username js-action-profile-name"), (item.IndexOf("div", item.IndexOf("username js-action-profile-name")) - item.IndexOf("username js-action-profile-name"))).Replace("username js-action-profile-name", string.Empty).Replace("\"", string.Empty).Replace("u003E", string.Empty).Replace("u003Cs", string.Empty).Replace("/s", string.Empty).Replace("u003Cb", string.Empty).Replace("/b", string.Empty).Replace("/span", string.Empty).Replace("n", string.Empty).Replace("/a", string.Empty).Replace("\\\\", string.Empty).Replace("\u003C\\\\", string.Empty).Replace("\u003C\\\u003C\\\\", string.Empty).Replace(@"\pa\\", string.Empty).Replace("\u003C\\\\", string.Empty).Replace("\\u003C\\/", string.Empty).Replace(@"\u003C", string.Empty).Replace(@"\u003C\u003C\pa", string.Empty).Replace(@"\u003C\", string.Empty).Replace(@"\", string.Empty).Trim();
//Testing
string findlastPa = string.Empty;
if (tweetUserName.Contains("pa"))
{
findlastPa = findlastPa + tweetUserName[tweetUserName.Length - 2].ToString();
findlastPa = findlastPa + tweetUserName[tweetUserName.Length - 1].ToString();
if (findlastPa == "pa")
{
tweetUserName = tweetUserName.Replace("pa", string.Empty).Trim();
}
}
}
catch
{
}
List<string> lst_TweetUserName = globusHttpHelper.GetTextDataByTagAndAttributeName(item, "span", "username js-action-profile-name");
if (lst_TweetUserName.Count > 0)
{
foreach (string item2 in lst_TweetUserName)
{
try
{
if (!string.IsNullOrEmpty(item2))
{
tweetUserName = item2;
break;
}
}
catch
{
}
}
}
}
catch
{
}
}
if (item.Contains("js-tweet-text"))
{
try
{
try
{
tweetMessage = item.Substring(item.IndexOf("js-tweet-text"), (item.IndexOf("div", item.IndexOf("js-tweet-text")) - item.IndexOf("js-tweet-text"))).Replace("js-tweet-text", string.Empty).Replace("\"", string.Empty).Replace("u003E", string.Empty).Replace("u003Cs", string.Empty).Replace("/s", string.Empty).Replace("u003Cb", string.Empty).Replace("/b", string.Empty).Replace("/span", string.Empty).Replace("n", string.Empty).Replace("/a", string.Empty).Replace("\\\\", string.Empty).Replace("\u003C\\\\", string.Empty).Replace("\u003C\\\u003C\\\\", string.Empty).Replace(@"\pa\\", string.Empty).Replace("\u003C\\\\", string.Empty).Replace("\\u003C\\/", string.Empty).Replace(@"\u003C", string.Empty).Replace(@"\u003C\u003C\pa", string.Empty).Replace(@"\u003C\", string.Empty).Replace(@"\", string.Empty).Trim();
if (tweetMessage.Contains("twitter-atreply pretty-lik dir=ltr "))
{
try
{
//int startIndex = tweetMessage.IndexOf("twitter-atreply pretty-lik");
tweetMessage = tweetMessage.Substring(tweetMessage.IndexOf("twitter-atreply pretty-lik dir=ltr"), (tweetMessage.Length) - (tweetMessage.IndexOf(@"twitter-atreply pretty-lik dir=ltr "))).Replace(@"twitter-atreply pretty-lik dir=ltr", string.Empty).Replace("\"", string.Empty).Replace("u003E", string.Empty).Replace("u003Cs", string.Empty).Replace("/s", string.Empty).Replace("u003Cb", string.Empty).Replace("/b", string.Empty).Replace("/span", string.Empty).Replace("n", string.Empty).Replace("/a", string.Empty).Replace("\\\\", string.Empty).Replace("\u003C\\\\", string.Empty).Replace("\u003C\\\u003C\\\\", string.Empty).Replace(@"\pa\\", string.Empty).Replace("\u003C\\\\", string.Empty).Replace("\\u003C\\/", string.Empty).Replace(@"\u003C", string.Empty).Replace(@"\u003C\u003C\pa", string.Empty).Replace(@"\u003C\", string.Empty).Replace(@"\", string.Empty).Trim();
}
catch
{
}
}
}
catch
{
}
List<string> lstTweetReply = globusHttpHelper.GetTextDataByTagAndAttributeName(item, "p", "js-tweet-text");
if (lstTweetReply.Count > 0)
{
foreach (string item3 in lstTweetReply)
{
try
{
if (!string.IsNullOrEmpty(item3))
{
tweetMessage = item3;
}
}
catch
{
}
}
}
}
catch
{
}
}
if (item.Contains("data-user-id="))
{
try
{
string tweetUserId1 = item.Substring(item.IndexOf("data-user-id=")+13, (item.IndexOf("\"", item.IndexOf("data-user-id=")+13) - item.IndexOf("data-user-id=")+13)).Replace("data-user-id=", string.Empty).Replace("\"", string.Empty).Trim();
string[] ArrtweetUserId = Regex.Split(tweetUserId1, "[^0-9]");
foreach (string item1 in ArrtweetUserId)
{
try
{
if (item1.Length > 3)
{
tweetId = item1;
break;
}
}
catch
{
}
}
}
catch
{
}
}
}
catch
{
}
DataSet ds2 = obj_clsDB_Randomiser.Selecttb_RandomiserMention(StringEncoderDecoder.Encode(UserName), StringEncoderDecoder.Encode(tweetUserName), StringEncoderDecoder.Encode(tweetMessage));
if (ds2.Tables["tb_RandomiserMention"].Rows.Count < 1)
{
if (!string.IsNullOrEmpty(tweetUserName)) // && tweetUserName !="@"+Screen_Name
{
obj_clsDB_Randomiser.Mention_InsertIntotb_RandomiserMention(StringEncoderDecoder.Encode(userName), StringEncoderDecoder.Encode(Screen_Name), StringEncoderDecoder.Encode(tweetId), StringEncoderDecoder.Encode(tweetUserName), StringEncoderDecoder.Encode(tweetMessage));
Log("[ " + DateTime.Now + " ] => [ " + count + "Record Saved in Database With User Name : " + UserName + " ]");
count++;
}
}
}
if (status.Contains("\"max_id\":"))
{
try
{
string max_id2 = status.Substring(status.IndexOf("\"max_id\":"), status.Length - status.IndexOf("\"max_id\":"));
string max_id1 = status.Substring(status.IndexOf("\"max_id\":") + 10, (status.IndexOf("\"", (status.IndexOf("\"max_id\":") + 10)) - (status.IndexOf("\"max_id\":") + 10))).Replace("\"max_id\":", string.Empty).Replace("\"", string.Empty).Trim();
string[] arrMaxId = Regex.Split(max_id, "[^0-9]");
foreach (string item in arrMaxId)
{
try
{
if (item.Length > 5)
{
max_id = max_id1;
GetMentions_ThroughAjax(ref globusHttpHelper, ref userId, ref userName, ref Screen_name, ref postAuthenticityToken, max_id);
//List<string> lstNextAjaxId = GetStatusIdThroughAjax(ref globusHttpHelper, screenName, item);
//break;
}
}
catch
{
}
}
}
catch
{
}
}
}
}
catch
{
}
}
public void GetMentions(ref Globussoft.GlobusHttpHelper globusHttpHelper, ref string userId, ref string userName, ref string Screen_name, ref string postAuthenticityToken)
{
try
{
string tweetId = string.Empty;
string tweetUserName = string.Empty;
string tweetMessage = string.Empty;
string maxId = string.Empty;
Log("[ " + DateTime.Now + " ] => [ Starting Scrap Mentions.............With User Name : " + UserName + " ]");
string pageSource = globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/mentions"), "", "");
DataSet ds_BeforeInsertionCount = obj_clsDB_Randomiser.SelectAllfromtb_RandomiserMention();
int countBeforeInsertion = ds_BeforeInsertionCount.Tables["tb_RandomiserMention"].Rows.Count;
if (pageSource.Contains("stream-container "))
{
string streamContainer = globusHttpHelper.GetDataWithTagValueByTagAndAttributeName(pageSource, "div", "stream-container ");
string[] arrContent=Regex.Split(streamContainer, "content");
foreach (string item in arrContent)
{
try
{
if (item.Contains("username js-action-profile-name"))
{
try
{
List<string> lst_TweetUserName = globusHttpHelper.GetTextDataByTagAndAttributeName(item, "span", "username js-action-profile-name");
if (lst_TweetUserName.Count > 0)
{
foreach (string item2 in lst_TweetUserName)
{
try
{
if(!string.IsNullOrEmpty(item2))
{
tweetUserName = item2;
break;
}
}
catch
{
}
}
}
}
catch
{
}
}
if (item.Contains("js-tweet-text"))
{
try
{
List<string> lstTweetReply = globusHttpHelper.GetTextDataByTagAndAttributeName(item, "p", "js-tweet-text");
if (lstTweetReply.Count > 0)
{
foreach (string item3 in lstTweetReply)
{
try
{
if (!string.IsNullOrEmpty(item3))
{
tweetMessage = item3;
}
}
catch
{
}
}
}
}
catch
{
}
}
if (item.Contains("data-user-id="))
{
try
{
string tweetUserId1 = item.Substring(item.IndexOf("data-user-id="), item.IndexOf(" ", item.IndexOf("data-user-id=")) - item.IndexOf("data-user-id=")).Replace("data-user-id=", string.Empty).Replace("\"", string.Empty).Trim();
string[] ArrtweetUserId = Regex.Split(tweetUserId1, "[^0-9]");
foreach (string item1 in ArrtweetUserId)
{
try
{
if (item1.Length > 3)
{
tweetId = item1;
}
}
catch
{
}
}
}
catch
{
}
}
if (item.Contains("data-max-id="))
{
try
{
string data_max_id = item.Substring(item.IndexOf("data-max-id="), item.IndexOf(" ", item.IndexOf("data-max-id=")) - item.IndexOf("data-max-id=")).Replace("data-max-id=", string.Empty).Replace("\"", string.Empty).Trim();
string[] arrdata_max_id = Regex.Split(data_max_id, "[^0-9]");
foreach (string item4 in arrdata_max_id)
{
try
{
if (item4.Length > 3)
{
maxId = item4;
}
}
catch
{
}
}
}
catch
{
}
}
}
catch
{
}
DataSet ds2 = obj_clsDB_Randomiser.Selecttb_RandomiserMention(UserName, tweetUserName, tweetMessage);
if (ds2.Tables["tb_RandomiserMention"].Rows.Count < 1)
{
if (!string.IsNullOrEmpty(tweetUserName))
{
obj_clsDB_Randomiser.Mention_InsertIntotb_RandomiserMention(StringEncoderDecoder.Encode(userName), StringEncoderDecoder.Encode(Screen_Name), StringEncoderDecoder.Encode(tweetId), StringEncoderDecoder.Encode(tweetUserName), StringEncoderDecoder.Encode(tweetMessage));
Log("[ " + DateTime.Now + " ] => [ " + count + "Record Saved in Database With User Name : " + UserName + " ]");
count++;
}
}
}
}
DataSet ds_AfterInsertionCount = obj_clsDB_Randomiser.SelectAllfromtb_RandomiserMention();
int countAfterInsertion = ds_AfterInsertionCount.Tables["tb_RandomiserMention"].Rows.Count;
if (countBeforeInsertion == countAfterInsertion)
{
Log("[ " + DateTime.Now + " ] => [ There Is No New Mentions With User Name : " + UserName + " ]");
}
/// Ajax Request
GetMentions_ThroughAjax(ref globusHttpHelper, ref userId, ref userName, ref Screen_name, ref postAuthenticityToken, maxId);
Log("[ " + DateTime.Now + " ] => [ PROCESS COMPLETED With User Name : " + UserName + " ]");
Log("------------------------------------------------------------------------------------------------------------------------------------------");
}
catch
{
}
}
