public List <string> GetAllBoardNames_new1(string screenName) { BaseLib.GlobusRegex rgx = new GlobusRegex(); Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper(); //GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Getting All Board Names ]"); string UserUrl = "http://pinterest.com/" + screenName; string BoardPage = httpHelper.getHtmlfromUrl(new Uri(UserUrl), "", "", ""); string[] data = Regex.Split(BoardPage, "is_collaborative"); foreach (var itemdata in data) { try { string boardUrl = Utils.Utils.getBetween(itemdata, ", \"url\":", ",").Replace("\"", "").Trim(); if (!lstBoardUrls.Contains(boardUrl) && !string.IsNullOrEmpty(boardUrl)) { lstBoardUrls.Add(boardUrl); } if (itemdata.Contains("board_id")) { string boardId = Utils.Utils.getBetween(itemdata, "board_id\":", ",").Replace("\"", "").Trim(); if (!lstBoardId.Contains(boardId)) { lstBoardId.Add(boardId); } } } catch (Exception ex) { GlobusLogHelper.log.Error("Error :" + ex.StackTrace); } } string[] Items = Regex.Split(BoardPage, "item"); int counter = 0; foreach (string item in Items) { try { if (item.Contains("id=\\\"Board") && item.Contains("boardLinkWrapper")) { //if (counter == 1) { string[] Data = System.Text.RegularExpressions.Regex.Split(item, "boardLinkWrapper"); foreach (string Dataitem in Data) { if (Dataitem.Contains("boardName")) { string BoardUrl = string.Empty; int startIndex = Dataitem.IndexOf("title"); int LastPoint = Dataitem.IndexOf("<h2"); string Board = Dataitem.Substring(startIndex, LastPoint).Replace("\\n", string.Empty).Replace("\"", "").Replace("<div class=\\b", string.Empty).Replace(" ", string.Empty).Replace("\"title\"", "").Replace("</div", ""); BoardUrl = rgx.StripTagsRegex(Board); try { Board = Utils.Utils.getBetween(BoardUrl, ">>", "<"); //modified done if (Board == "") { Board = Utils.Utils.getBetween(BoardUrl, "title=", ">").Replace("\\", "").Trim(); } } catch (Exception ex) { }; if (!BoardNames.Contains(Board)) { BoardNames.Add(Board); //DropDowenBox.Items.Add(Board); } } } } counter++; } } catch (Exception ex) { GlobusLogHelper.log.Error("Error :" + ex.StackTrace); } } return(BoardNames); }
public List<StructTweetIDs> GetTweetData_New_ForCampaign(string keyword,int noOfReplies) { try { int counter = 0; int counterNoOfReplies = noOfReplies; lst_structTweetIDs = new List<StructTweetIDs>(); string res_Get_searchURL = string.Empty; string searchURL = string.Empty; string maxid = string.Empty; string TweetId = string.Empty; if (keyword.Trim().Contains(" ")) { keyword = keyword.Replace(" ", "+"); } startAgain: //if (counter == 0) //{ // searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + ""; // counter++; //} //else //{ // searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + "&scroll_cursor=" + TweetId; //} if (counter == 0) { //searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + ""; searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime"; counter++; } else { //searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + "&scroll_cursor=" + TweetId; searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; } try { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); if (string.IsNullOrEmpty(res_Get_searchURL)) { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } try { //string sjss = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor"); //splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("refresh_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } if (item.Contains("scroll_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } } } catch (Exception) { } } catch (Exception ex) { System.Threading.Thread.Sleep(2000); res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs); } if (!string.IsNullOrEmpty(res_Get_searchURL)) { object DEserizedData = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL); string DataHtml = (string)((JObject)DEserizedData)["items_html"]; string[] splitRes = Regex.Split(DataHtml, "data-item-id"); splitRes = splitRes.Skip(1).ToArray(); GlobusRegex regx = new GlobusRegex(); foreach (string item in splitRes) { if (item.Contains("data-screen-name=") && !item.Contains("follow-button") && !item.Contains("Following")) { } else { continue; } string modified_Item = "\"from_user\"" + item; string id = ""; try { int startIndex = item.IndexOf("data-user-id="); string start = item.Substring(startIndex).Replace("data-user-id=", ""); int endIndex = start.IndexOf("data-is-reply-to"); if (endIndex == -1) { endIndex = start.IndexOf("data-expanded-footer="); } if (endIndex == -1) { endIndex = start.IndexOf(">"); } string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "").Replace("\n", string.Empty); if (end.Contains(" ")) { end = end.Split(' ')[0]; } id = end; } catch (Exception ex) { id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } string from_user_id = ""; try { int startIndex = item.IndexOf("data-screen-name="); string start = item.Substring(startIndex).Replace("data-screen-name=", ""); int endIndex = start.IndexOf("data-name"); if (endIndex > 100) { endIndex = 0; endIndex = start.IndexOf("data-user"); } string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); from_user_id = end; } catch (Exception ex) { from_user_id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } string tweetUserid = string.Empty; try { int startIndex = item.IndexOf("=\""); string start = item.Substring(startIndex).Replace("=\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end; } catch (Exception ex) { from_user_id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } ///Tweet Text string text = string.Empty; try { int startindex = item.IndexOf("js-tweet-text tweet-text\""); if (startindex == -1) { startindex = 0; startindex = item.IndexOf("js-tweet-text tweet-text"); } string start = item.Substring(startindex).Replace("js-tweet-text tweet-text\"", "").Replace("js-tweet-text tweet-text tweet-text-rtl\"", ""); int endindex = start.IndexOf("</p>"); if (endindex == -1) { endindex = 0; endindex = start.IndexOf("stream-item-footer"); } string end = start.Substring(0, endindex); end = regx.StripTagsRegex(end); text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); string[] array = Regex.Split(text, "http"); text = string.Empty; foreach (string itemData in array) { if (!itemData.Contains("t.co")) { string data = string.Empty; if (itemData.Contains("//")) { data = ("http" + itemData).Replace(" span ", string.Empty); if (!text.Contains(itemData.Replace(" ", "")))// && !data.Contains("class") && !text.Contains(data)) { text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("lang=en data-aria-label-part=0",string.Empty); } } else { if (!text.Contains(itemData.Replace(" ", ""))) { text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("lang=en data-aria-label-part=0",string.Empty); } } } } if (text.Contains("data-aria-label-part=0")) { text = globushttpHelper.getBetween(text + ":&$#@", "data-aria-label-part=0", ":&$#@"); } } catch { }; StructTweetIDs structTweetIDs = new StructTweetIDs(); if (id != "null") { structTweetIDs.ID_Tweet = tweetUserid; structTweetIDs.ID_Tweet_User = id; structTweetIDs.username__Tweet_User = from_user_id; structTweetIDs.wholeTweetMessage = text; lst_structTweetIDs.Add(structTweetIDs); } if (lst_structTweetIDs.Count >= counterNoOfReplies) { return lst_structTweetIDs; } } lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList(); } if (lst_structTweetIDs.Count > counterNoOfReplies) { if (res_Get_searchURL.Contains("has_more_items\":false")) { return lst_structTweetIDs; } else goto startAgain; } else { if (res_Get_searchURL.Contains("has_more_items\":false")) { return lst_structTweetIDs; } else goto startAgain; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return lst_structTweetIDs; }
public string countNoOfTweet(string keyword) { //GlobusHttpHelper globushttpHelper = new GlobusHttpHelper(); GlobusRegex rgx = new GlobusRegex(); string user_name = string.Empty; string PageSource = string.Empty; string TweetCount = string.Empty; PageSource = globushttpHelper.getHtmlfromUrl(new Uri("http://www.twitter.com/" + keyword), " ", " "); if (!PageSource.Contains("Account suspended") && !PageSource.Contains("currently suspended") && !PageSource.Contains("account-suspended")) { try { int indexStart = PageSource.IndexOf("data-element-term=\"tweet_stats"); string start = PageSource.Substring(indexStart).Replace("data-element-term=\"tweet_stats", ""); int indexEnd = start.IndexOf("</strong>"); string end = start.Substring(0, indexEnd).Replace("<strong>", "").Replace(">", "").Replace("\n", "").Replace(" ", ""); if (end.Contains("<strongtitle=")) { try { end = end.Split('\"')[6].Replace("strongtitle=", ""); } catch { } } TweetCount = rgx.StripTagsRegex(end).Replace("data-nav=\"tweet\"", string.Empty).Replace("\n", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim(); } catch { } if (string.IsNullOrEmpty(TweetCount)) { try { int indexStart = PageSource.IndexOf("statuses_count"); string start = PageSource.Substring(indexStart).Replace("statuses_count", ""); int indexEnd = start.IndexOf(","); string end = start.Substring(0, indexEnd).Replace(""", "").Replace(":", "").Replace(",", "").Replace(";", "").Replace("/","").Replace("\"","").Trim(); TweetCount = rgx.StripTagsRegex(end).Replace("data-nav=\"tweet\"", string.Empty).Replace("\n", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim(); } catch { } } } return TweetCount; }
private void button5_Click(object sender, EventArgs e) { //Settings.Instance.MakeNewIeInstanceVisible = false; Settings.Instance.AutoMoveMousePointerToTopLeft = false; Settings.Instance.AutoStartDialogWatcher = false; Settings.Instance.WaitForCompleteTimeOut = 30000; GlobusRegex globusRegex = new GlobusRegex(); Thread newThread = new Thread(new ThreadStart(SignupPage)); newThread.SetApartmentState(ApartmentState.STA); newThread.Start(); AddToProxyAccountCreationLog("Strating Account Creation"); //new Thread(() => //{ // SignupPage(); //}).Start(); }
public List<string> GetAllBoardNames_new(ref PinInterestUser objPinInUser) { List<string> BoardNames = new List<string>(); BaseLib.GlobusRegex rgx = new GlobusRegex(); Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper(); string UserUrl = "http://pinterest.com/" + objPinInUser.Username; string BoardPage = httpHelper.getHtmlfromUrl(new Uri(UserUrl), "", "", objPinInUser.UserAgent); string[] Items = Regex.Split(BoardPage, "item"); int counter = 0; foreach (string item in Items) { try { if (item.Contains("id=\\\"Board") && item.Contains("boardLinkWrapper")) { //if (counter == 1) { string[] Data = System.Text.RegularExpressions.Regex.Split(item, "boardLinkWrapper"); foreach (string Dataitem in Data) { if (Dataitem.Contains("-end-")) { continue; } if (Dataitem.Contains("boardName")) { int startIndex = Dataitem.IndexOf("title"); int LastPoint = Dataitem.IndexOf("<h4"); string Board = Dataitem.Substring(startIndex, LastPoint).Replace("\\n", string.Empty).Replace("\"", "").Replace("<div class=\\b", string.Empty).Replace(" ", string.Empty).Replace("\"title\"", "").Replace("</div", ""); Board = rgx.StripTagsRegex(Board); try { Board = Board.Split('>')[1]; } catch { } if (!BoardNames.Contains(Board)) { BoardNames.Add(Board); } } } } counter++; } } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } } return BoardNames; }
public void StartKeywordExtracting(string keyword, int count) { try { keyword = keyword.Replace("#", "%23"); List<string> lstweete = new List<string>(); keyword = keyword.Replace("%23","#"); AddToTweetCreatorLogs("[ " + DateTime.Now + " ] => [ Extracting Tweets for " + keyword + " ]"); keyword = keyword.Replace("#", "%23"); string[] arraylst = new string[] { }; string scroll_cursor = "0"; GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); for (int i = 0; i < count; i++) { AddToTweetCreatorLogs("[ " + DateTime.Now + " ] => [ Getting " + (i + 1) + " Page Tweets ]"); string pgsrcs = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&scroll_cursor=" + scroll_cursor), "", ""); try { int startindex = pgsrcs.IndexOf("scroll_cursor"); string start = pgsrcs.Substring(startindex).Replace("scroll_cursor", string.Empty); int endindex = start.IndexOf("refresh_cursor"); string end = string.Empty; if (endindex >= 0) { end = start.Substring(0, endindex); scroll_cursor = end.Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); } else { endindex = start.IndexOf("\"}"); end = start.Substring(0, endindex); scroll_cursor = end; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> StartKeywordExtracting() --> Getting Maxid --> " + ex.Message, Globals.Path_TweetCreatorErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> StartKeywordExtracting() --> Getting Maxid --> " + ex.Message, Globals.Path_TwtErrorLogs); } arraylst = Regex.Split(pgsrcs, "tweet-text"); arraylst = arraylst.Skip(1).ToArray(); foreach (string data in arraylst) { try { int stratindex = data.IndexOf("\""); string start = data.Substring(stratindex); Regex regex = new Regex(@"\\u([0-9a-z]{4})", RegexOptions.IgnoreCase); start = regex.Replace(start, match => char.ConvertFromUtf32(Int32.Parse(match.Groups[1].Value, System.Globalization.NumberStyles.HexNumber))); string abc = start.Substring(start.IndexOf(">"), start.IndexOf("<div") - start.IndexOf(">") - 1); BaseLib.GlobusRegex baselib = new GlobusRegex(); string TweetText = baselib.StripTagsRegex(abc); try { if (TweetText.Contains("http:\\") || TweetText.Contains("pic.twitter.com")) { if (TweetText.Contains("http:\\")) { string[] array = Regex.Split(TweetText, "http:"); TweetText = array[0]; } else if (TweetText.Contains("pic.twitter.com")) { string[] array = Regex.Split(TweetText, "pic.twitter.com"); TweetText = array[0]; } } TweetText = TweetText.Replace(""", string.Empty).Replace(">", string.Empty).Replace("<", string.Empty).Replace("&", string.Empty).Replace("\n", string.Empty).Replace("\\n", string.Empty).Replace(" ", string.Empty).Replace(">", string.Empty).Replace("'", string.Empty).Replace("/", "");// \n \n if (!string.IsNullOrEmpty(TweetText) && TweetText.Length <= 100) { lstweete.Add(TweetText); } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> StartKeywordExtracting() --> RemovingLinks --> " + ex.Message, Globals.Path_TweetCreatorErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> StartKeywordExtracting() --> RemovingLinks --> " + ex.Message, Globals.Path_TwtErrorLogs); } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> StartKeywordExtracting() --> Getting Tweet Message --> " + ex.Message, Globals.Path_TweetCreatorErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> StartKeywordExtracting() --> Getting Tweet Message --> " + ex.Message, Globals.Path_TwtErrorLogs); } } } lstweete = lstweete.Distinct().ToList(); AddToTweetCreatorLogs("[ " + DateTime.Now + " ] => [ " + lstweete.Count + " Total distinct Tweets ]"); foreach (string Tweet in lstweete) { AddToTweetCreatorLogs("[ " + DateTime.Now + " ] => [ " + Tweet + " ]"); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(Tweet, Globals.path_StoreKeywordTweetExtractor + "_" + keyword.Replace("%23", "#") + ".txt"); } keyword = keyword.Replace("%23", "#"); AddToTweetCreatorLogs("[ " + DateTime.Now + " ] => [ Finished Extracting Tweets for " + keyword + " ]"); AddToTweetCreatorLogs("-----------------------------------------------------------------------------------------------------------------------"); } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> StartKeywordExtracting() --> " + ex.Message, Globals.Path_TweetCreatorErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> StartKeywordExtracting() --> " + ex.Message, Globals.Path_TwtErrorLogs); } }
public List<StructTweetIDs> GetTweetData_New(string keyword) { lst_structTweetIDs = new List<StructTweetIDs>(); string Nextcounter = "0"; try { //StartAgain: string searchURL = "https://twitter.com/i/search/timeline?type=relevance&src=typd&include_available_features=1&include_entities=1&max_id=" + noOfRecords + "&q=" + Uri.EscapeDataString(keyword); string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), string.Empty, string.Empty); JObject Abc = JObject.Parse(res_Get_searchURL); string datahkj = string.Empty; datahkj = Abc["items_html"].ToString(); string[] splitRes = Regex.Split(((string)Abc["items_html"]), "js-stream-item stream-item stream-item expanding-stream-item");//Regex.Split(res_Get_searchURL, "{\"created_at\""); splitRes = splitRes.Skip(1).ToArray(); GlobusRegex regx = new GlobusRegex(); foreach (string item in splitRes) { string from_user = string.Empty; string from_user_id = string.Empty; string from_user_name = string.Empty; string id = string.Empty; string text = string.Empty; ///Tweet ID try { int startindex = item.IndexOf("data-item-id=\""); string start = item.Substring(startindex).Replace("data-item-id=\"", ""); int endindex = start.IndexOf("\""); string end = start.Substring(0, endindex); id = end; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> userid --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> userid --> " + ex.Message, Globals.Path_TwtErrorLogs); } ///Tweet User Screen name try { int startindex = item.IndexOf("data-screen-name=\""); string start = item.Substring(startindex).Replace("data-screen-name=\"", ""); int endindex = start.IndexOf("\""); string end = start.Substring(0, endindex); from_user_name = end; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> from_user_name --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> from_user_name --> " + ex.Message, Globals.Path_TwtErrorLogs); } ///Tweet User User-id try { int startindex = item.IndexOf("data-user-id=\""); string start = item.Substring(startindex).Replace("data-user-id=\"", ""); int endindex = start.IndexOf("\""); string end = start.Substring(0, endindex); from_user_id = end; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } ///Tweet Text try { int startindex = item.IndexOf("js-tweet-text tweet-text\""); string start = item.Substring(startindex).Replace("js-tweet-text tweet-text\"", ""); int endindex = start.IndexOf("</p>"); string end = start.Substring(0, endindex); end = regx.StripTagsRegex(end); text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); string[] array = Regex.Split(text, "http"); text = string.Empty; foreach (string itemData in array) { if (!itemData.Contains("t.co")) { string data = string.Empty; if (itemData.Contains("//")) { data = "http" + itemData; if (!text.Contains(itemData.Replace(" ", ""))) { text += data; } } else { if (!text.Contains(itemData.Replace(" ", ""))) { text += itemData; } } } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwtErrorLogs); } StructTweetIDs structTweetIDs = new StructTweetIDs(); structTweetIDs.ID_Tweet = id; structTweetIDs.ID_Tweet_User = from_user_id; structTweetIDs.username__Tweet_User = from_user_name; structTweetIDs.wholeTweetMessage = text; Log("[ " + DateTime.Now + " ] => [ " + id + " ]"); Log("[ " + DateTime.Now + " ] => [ " + from_user_id + " ]"); Log("[ " + DateTime.Now + " ] => [ " + from_user_name + " ]"); Log("[ " + DateTime.Now + " ] => [ " + text + " ]"); Log("---------------------------------------------------------------------------------------------------------------------------------------------------"); lst_structTweetIDs.Add(structTweetIDs); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(id + ":" + from_user_id, Globals.Path_keywordFollowerScrapedData); } //if (res_Get_searchURL.Contains("\"has_more_items\":true")) //{ // try // { // int startindex = res_Get_searchURL.IndexOf("{\"max_id\":\""); // string start = res_Get_searchURL.Substring(startindex).Replace("{\"max_id\":\"", ""); // int endindex = start.IndexOf("\","); // string end = start.Substring(0, endindex); // Nextcounter = end; // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs); // } // //goto StartAgain; //} return lst_structTweetIDs; } catch (Exception ex) { return lst_structTweetIDs; GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } }
public List<string> UserPins_Repin(string UserName, ref PinInterestUser objPinUser) { List<string> lstUsernamePin = new List<string>(); try { GlobusRegex objGlobusRegex = new GlobusRegex(); GlobusHttpHelper objglobusHttpHelper = new GlobusHttpHelper(); GlobusLogHelper.log.Info("[ " + DateTime.Now + "] => [ Start Getting Pins For this User " + UserName + " ]"); string UserPins = string.Empty; if (UserName.Contains("pinterest.com")) { UserUrl = UserName; UserPins = UserName + "pins/"; } else { UserUrl = "https://pinterest.com/" + UserName + "/"; UserPins = "https://pinterest.com/" + UserName + "/pins/"; } try { UserPageSource = objglobusHttpHelper.getHtmlfromUrl(new Uri(UserUrl), "http://pinterest.com/", string.Empty, objPinUser.UserAgent); UserPinPageSource = objglobusHttpHelper.getHtmlfromUrl(new Uri(UserPins), UserUrl, string.Empty, objPinUser.UserAgent); } catch (Exception ex) { } List<string> lst = objGlobusRegex.GetHrefUrlTagsForPinDescription(UserPinPageSource); lst = lst.Distinct().ToList(); string PinUrl = string.Empty; foreach (string item in lst) { try { if (item.Contains("/pin/")) { if (!item.Contains("/pin/A")) { try { int FirstPinPoint = item.IndexOf("/pins/"); int SecondPinPoint = item.IndexOf("class="); PinUrl = Utils.Utils.getBetween(item, "/pin/", "/\" class="); if (!string.IsNullOrEmpty(PinUrl)) { lstUsernamePin.Add(PinUrl); } } catch (Exception ex) { PinUrl = item.Replace("/pins/", ""); lstUsernamePin.Add(PinUrl); } } } } catch (Exception ex) { } } lstUsernamePin = lstUsernamePin.Distinct().ToList(); lstUsernamePin.Reverse(); GlobusLogHelper.log.Info("[ " + DateTime.Now + "] => [ Total Pin Urls Collected " + lstUsernamePin.Count + " ]"); } catch (Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } return lstUsernamePin; }
public List<StructTweetIDs> NewKeywordStructDataSearchByPeople(string keyword) { try { BaseLib.GlobusRegex regx = new GlobusRegex(); int counter = 0; lst_structTweetIDs = new List<StructTweetIDs>(); string res_Get_searchURL = string.Empty; string searchURL = string.Empty; string maxid = string.Empty; string TweetId = string.Empty; string text = string.Empty; string ProfileName = string.Empty; string Location = string.Empty; string Bio = string.Empty; string website = string.Empty; string NoOfTweets = string.Empty; string Followers = string.Empty; string Followings = string.Empty; if (keyword.Trim().Contains(" ")) { keyword = keyword.Replace(" ", "+"); } startAgain: if (!RetweetFromUserName) { if (counter == 0) { //searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + ""; //searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime"; //29-4-2014 only for client it has been changed searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&mode=users"; counter++; } else { //searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; //29-4-2014 only for client it is changed searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&mode=users&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; } } else { searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + RetweetExtractcount + "&include_available_features=1&include_entities=1"; } try { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); if (string.IsNullOrEmpty(res_Get_searchURL)) { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } try { //string sjss = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor"); //splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("refresh_cursor")) { int startIndex = item.IndexOf("USER-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } if (item.Contains("scroll_cursor")) { int startIndex = item.IndexOf("USER-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } } } catch (Exception) { } } catch (Exception ex) { System.Threading.Thread.Sleep(2000); res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs); } // && !res_Get_searchURL.Contains("has_more_items\":false") if (!string.IsNullOrEmpty(res_Get_searchURL)) { //string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); //Regex.Split(res_Get_searchURL, "\"in_reply_to_status_id_str\""); string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("data-screen-name=") && !item.Contains("js-actionable-user js-profile-popup-actionable")) { //var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL); //string DataHtml = (string)avc["items_html"]; } else { //continue; } string modified_Item = "\"from_user\"" + item; string id = ""; try { int startIndex = item.IndexOf("data-user-id="); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); id = end; } catch (Exception ex) { id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } string from_user_id = ""; try { int startIndex = item.IndexOf("data-screen-name=\\\""); string start = item.Substring(startIndex).Replace("data-screen-name=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); from_user_id = end; } catch (Exception ex) { from_user_id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } string tweetUserid = string.Empty; try { int startIndex = item.IndexOf("=\\\""); string start = item.Substring(startIndex).Replace("=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end; } catch (Exception ex) { from_user_id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } StructTweetIDs structTweetIDs = new StructTweetIDs(); if (id != "null") { structTweetIDs.ID_Tweet = tweetUserid; structTweetIDs.ID_Tweet_User = id; structTweetIDs.username__Tweet_User = from_user_id; structTweetIDs.wholeTweetMessage = text; lst_structTweetIDs.Add(structTweetIDs); Log("[ " + DateTime.Now + " ] => [ " + tweetUserid + " ]"); Log("[ " + DateTime.Now + " ] => [ " + id + " ]"); Log("[ " + DateTime.Now + " ] => [ " + from_user_id + " ]"); Log("-------------------------------------------------------------------------------------------------------------------------------"); if (!File.Exists(Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv")) { GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS", Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"); } //foreach (TwitterDataScrapper.StructTweetIDs item in data) { ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr(); GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + from_user_id), "", ""); string Responce = ProfilePageSource; #region Convert HTML to XML string xHtml = objChilkat.ConvertHtmlToXml(Responce); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion int counterdata = 0; xBeginSearchAfter = null; string dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterdata == 0) { ProfileName = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else if (counterdata == 1) { website = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else { break; } // xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor"); } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim(); break; } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Location = xNode.AccumulateTagContent("text", "script|style"); break; } int counterData = 0; xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterData == 0) { // NoOfTweets = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileNav-value"); NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty); counterData++; } else if (counterData == 1) { Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty); counterData++; } else if (counterData == 2) { Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty); counterData++; } else { break; } //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor"); } if (!string.IsNullOrEmpty(from_user_id) && tweetUserid != "null") { string Id_user = tweetUserid.Replace("}]", string.Empty).Trim(); Globals.lstScrapedUserIDs.Add(Id_user); GlobusFileHelper.AppendStringToTextfileNewLine(id + "," + from_user_id + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"); GlobusFileHelper.AppendStringToTextfileNewLine(from_user_id, Globals.Path_KeywordScrapedListData + "-" + keyword + ".txt"); Log("[ " + DateTime.Now + " ] => [ " + from_user_id + "," + Id_user + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]"); } } } lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList(); if (lst_structTweetIDs.Count >= noOfRecords) { return lst_structTweetIDs; } } if (lst_structTweetIDs.Count <= noOfRecords) { maxid = lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet; if (res_Get_searchURL.Contains("has_moreitems\":false")) { return lst_structTweetIDs; } else { goto startAgain; } } else { if (res_Get_searchURL.Contains("has_more_items\":false")) { return lst_structTweetIDs; } else goto startAgain; } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return lst_structTweetIDs; }
public List<StructTweetIDs> NewKeywordStructDataForOnlyTweet(string keyword) { try { BaseLib.GlobusRegex regx = new GlobusRegex(); int counter = 0; lst_structTweetIDs = new List<StructTweetIDs>(); //lstTweetIds=new List<string>(); string res_Get_searchURL = string.Empty; string searchURL = string.Empty; string maxid = string.Empty; string TweetId = string.Empty; string text = string.Empty; if (keyword.Trim().Contains(" ")) { keyword = keyword.Replace(" ", "+"); } startAgain: if (!RetweetFromUserName) { if (counter == 0) { searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime"; counter++; } else { searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; } } else { searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + RetweetExtractcount + "&include_available_features=1&include_entities=1"; } try { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); if (string.IsNullOrEmpty(res_Get_searchURL)) { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } try { string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor"); foreach (string item in splitRes) { if (item.Contains("refresh_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } if (item.Contains("scroll_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } } } catch (Exception) { } } catch (Exception ex) { System.Threading.Thread.Sleep(2000); res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs); } // && !res_Get_searchURL.Contains("has_more_items\":false") if (!string.IsNullOrEmpty(res_Get_searchURL)) { //string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); //Regex.Split(res_Get_searchURL, "\"in_reply_to_status_id_str\""); string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("data-screen-name=") && !item.Contains("js-actionable-user js-profile-popup-actionable")) { } else { continue; } string modified_Item = "\"from_user\"" + item; string id = ""; try { int startIndex = item.IndexOf("data-user-id="); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); id = end; } catch (Exception ex) { id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } string tweetUserid = string.Empty; try { int startIndex = item.IndexOf("=\\\""); string start = item.Substring(startIndex).Replace("=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end; } catch (Exception ex) { tweetUserid = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } try { int startindex = item.IndexOf("js-tweet-text tweet-text"); string start = item.Substring(startindex).Replace("js-tweet-text tweet-text", ""); int endindex = start.IndexOf("</p>"); if (endindex == -1) { endindex = 0; endindex = start.IndexOf("stream-item-footer"); } string end = start.Substring(0, endindex); end = regx.StripTagsRegex(end); text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "").Replace("dir=\"ltr\"", ""); text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", "").Replace("&", "&").Replace("'", "'").Replace("<", "<").Replace(">", ">"); //string[] array = Regex.Split(text, "http"); //text = string.Empty; //foreach (string itemData in array) //{ // if (!itemData.Contains("t.co")) // { // string data = string.Empty; // if (itemData.Contains("//")) // { // data = ("http" + itemData).Replace(" span ", string.Empty); // if (!text.Contains(itemData.Replace(" ", "")))// && !data.Contains("class") && !text.Contains(data)) // { // text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty); // } // } // else // { // if (!text.Contains(itemData.Replace(" ", ""))) // { // text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty); // } // } // } //} } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwtErrorLogs); } StructTweetIDs structTweetIDs = new StructTweetIDs(); if (!IsRetweetWithFovieteWithImages) { if (id != "null") { structTweetIDs.ID_Tweet = tweetUserid; structTweetIDs.ID_Tweet_User = id; lst_structTweetIDs.Add(structTweetIDs); //Log("[ " + DateTime.Now + " ] => [ " + tweetUserid + " ]"); //Log("-------------------------------------------------------------------------------------------------------------------------------"); } lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList(); queTweetId.Enqueue(tweetUserid); } else { if (text.Contains("http://t.co")) { if (id != "null") { structTweetIDs.ID_Tweet = tweetUserid; structTweetIDs.ID_Tweet_User = id; lst_structTweetIDs.Add(structTweetIDs); //Log("[ " + DateTime.Now + " ] => [ " + tweetUserid + " ]"); //Log("-------------------------------------------------------------------------------------------------------------------------------"); } lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList(); queTweetId.Enqueue(tweetUserid); } } //lstTweetIds.Add(tweetUserid); //lstTweetIds = lstTweetIds.Distinct().ToList(); //if (lst_structTweetIDs.Count >= noOfRecords) //{ // return lst_structTweetIDs; //} } if (lst_structTweetIDs.Count <= noOfRecords) { maxid = lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet; if (res_Get_searchURL.Contains("has_moreitems\":false")) { return lst_structTweetIDs; } else { goto startAgain; } } else { if (res_Get_searchURL.Contains("has_more_items\":false")) { return lst_structTweetIDs; } else goto startAgain; } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return lst_structTweetIDs; }
public List<StructTweetIDs> NewKeywordStructData1(string keyword) { try { BaseLib.GlobusRegex regx = new GlobusRegex(); int counter = 0; lst_structTweetIDs = new List<StructTweetIDs>(); string res_Get_searchURL = string.Empty; string searchURL = string.Empty; string maxid = string.Empty; string TweetId = string.Empty; string text = string.Empty; if (keyword.Trim().Contains(" ")) { keyword = keyword.Replace(" ", "+"); } startAgain: #region <Old Get request URl>> //if (counter == 0) //{ // //searchURL = "https://twitter.com/i/search/realtime?type=relevance&src=typd&include_available_features=1&include_entities=1&q=" + Uri.EscapeDataString(keyword); // //searchURL = "https://twitter.com/i/search/realtime?type=relevance&src=typd&composed_count=0&count=" + noOfRecords + "&include_available_features=1&include_entities=1&max_id=" + maxid + "&q=" + keyword; // //searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + ""; // //counter++; // //searchURL = "https://twitter.com/search?q=ranbir&mode=relevance&src=typd"; // //searchURL = "https://twitter.com/search?q=" + keyword + "&mode=relevance&src=typd"; // searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&mode=relevance&composed_count=0&include_available_features=1&include_entities=1&include_new_items_bar=true&interval=30000"; // counter++; //} //else //{ // if (res_Get_searchURL.Contains("has_more_items\":false")) // { // return lst_structTweetIDs; // } // //searchURL = "https://twitter.com/i/search/timeline?type=relevance&src=typd&include_available_features=1&include_entities=1&max_id=" + maxid + "&q=" + keyword; // //searchURL = "https://twitter.com/i/search/timeline?q=ranbir&src=typd&mode=relevance&composed_count=0&include_available_features=1&include_entities=1&include_new_items_bar=true&interval=30000&latent_count=25&refresh_cursor=TWEET-372646612473876480-374800735134687234"; // searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&mode=relevance&composed_count=0&include_available_features=1&include_entities=1&include_new_items_bar=true&interval=30000&latent_count=25&refresh_cursor=" + TweetId; //} #endregion if (!RetweetFromUserName) { if (counter == 0) { //searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + ""; searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime"; counter++; } else { searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; //29-4-2014 only for client it is changed //searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime&mode=users&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; } } else { searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + RetweetExtractcount + "&include_available_features=1&include_entities=1"; } try { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); if (string.IsNullOrEmpty(res_Get_searchURL)) { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } try { //string sjss = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor"); //splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("refresh_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; ////only for client 29/4 ////int startIndex = item.IndexOf("TWEET-"); //int startIndex = item.IndexOf("USER-"); //string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); //int endIndex = start.IndexOf("\""); //string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); //TweetId = end; } if (item.Contains("scroll_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } } } catch (Exception) { } } catch (Exception ex) { System.Threading.Thread.Sleep(2000); res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs); } // && !res_Get_searchURL.Contains("has_more_items\":false") if (!string.IsNullOrEmpty(res_Get_searchURL)) { //string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); //Regex.Split(res_Get_searchURL, "\"in_reply_to_status_id_str\""); string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("data-screen-name=") && !item.Contains("js-actionable-user js-profile-popup-actionable")) { //var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL); //string DataHtml = (string)avc["items_html"]; } else { continue; } string modified_Item = "\"from_user\"" + item; string id = ""; try { int startIndex = item.IndexOf("data-user-id="); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); id = end; } catch (Exception ex) { id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } string from_user_id = ""; try { int startIndex = item.IndexOf("data-screen-name=\\\""); string start = item.Substring(startIndex).Replace("data-screen-name=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); from_user_id = end; } catch (Exception ex) { from_user_id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } string tweetUserid = string.Empty; try { int startIndex = item.IndexOf("=\\\""); string start = item.Substring(startIndex).Replace("=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end; } catch (Exception ex) { from_user_id = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } ///Tweet Text try { #region previous Code of Find Text //int startindex = item.IndexOf("js-tweet-text tweet-text\""); //TweetTextSize js-tweet-text tweet-text //if (startindex == -1) //{ // startindex = 0; // startindex = item.IndexOf("js-tweet-text tweet-text"); //} //string start = item.Substring(startindex).Replace("js-tweet-text tweet-text\"", "").Replace("js-tweet-text tweet-text tweet-text-rtl\"", ""); //int endindex = start.IndexOf("</p>"); //if (endindex == -1) //{ // endindex = 0; // endindex = start.IndexOf("stream-item-footer"); //} //string end = start.Substring(0, endindex); //end = regx.StripTagsRegex(end); //text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); //text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); //string[] array = Regex.Split(text, "http"); //text = string.Empty; //foreach (string itemData in array) //{ // if (!itemData.Contains("t.co")) // { // string data = string.Empty; // if (itemData.Contains("//")) // { // data = ("http" + itemData).Replace(" span ", string.Empty); // if (!text.Contains(itemData.Replace(" ", "")))// && !data.Contains("class") && !text.Contains(data)) // { // text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty); // } // } // else // { // if (!text.Contains(itemData.Replace(" ", ""))) // { // text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("js-tweet-text tweet-text", ""); // } // } // } //} #endregion string[] getTweetText = Regex.Split(item, "TweetTextSize js-tweet-text tweet-text"); try { text = Utils.getBetween(getTweetText[1], "data-aria-label-part=", "href="); text = text.Replace("\\\"0\\\"\\u003e", "").Replace("\\u003ca", ""); } catch { }; } catch { }; StructTweetIDs structTweetIDs = new StructTweetIDs(); if(text.Contains("")) if (id != "null") { structTweetIDs.ID_Tweet = tweetUserid; structTweetIDs.ID_Tweet_User = id; structTweetIDs.username__Tweet_User = from_user_id; structTweetIDs.wholeTweetMessage = text; lst_structTweetIDs.Add(structTweetIDs); Log("[ " + DateTime.Now + " ] => [ " + tweetUserid + " ]"); Log("[ " + DateTime.Now + " ] => [ " + id + " ]"); Log("[ " + DateTime.Now + " ] => [ " + from_user_id + " ]"); Log("-------------------------------------------------------------------------------------------------------------------------------"); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(id + ":" + from_user_id, Globals.Path_keywordFollowerScrapedData); } lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList(); if (lst_structTweetIDs.Count >= noOfRecords) { return lst_structTweetIDs; } } if (lst_structTweetIDs.Count <= noOfRecords) { maxid = lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet; if (res_Get_searchURL.Contains("has_moreitems\":false")) { return lst_structTweetIDs; } else { goto startAgain; } } else { if (res_Get_searchURL.Contains("has_more_items\":false")) { return lst_structTweetIDs; } else goto startAgain; } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } return lst_structTweetIDs; }
public void GetFollowercount() { try { Thread.CurrentThread.CurrentUICulture = new System.Globalization.CultureInfo("en"); GlobusRegex rgx = new GlobusRegex(); string URL = string.Empty; string PageSource = globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + Screen_name), "", ""); URL = globusHttpHelper.gResponse.ResponseUri.AbsoluteUri; if (PageSource.Contains("btn resend-confirmation-email-link")) { Isnonemailverifiedaccounts = true; } if (!PageSource.Contains("Account suspended") && !PageSource.Contains("currently suspended") && !URL.Contains("https://twitter.com/account/suspended") && !PageSource.Contains("account-suspended")) { //int indexStart = PageSource.IndexOf("data-nav=\"followers\""); //string start = PageSource.Substring(indexStart).Replace("data-nav=\'followers\'", ""); //int indexEnd = start.IndexOf("</strong>"); //string end = start.Substring(0, indexEnd).Replace("<strong>", "").Replace(">", "").Replace("\n", "").Replace(" ", ""); if (PageSource.Contains("data-element-term=\"follower_stats")) { try { int indexStart = PageSource.IndexOf("data-element-term=\"follower_stats"); string start = PageSource.Substring(indexStart).Replace("data-element-term=\"follower_stats", ""); int indexEnd = start.IndexOf("</strong>"); string end = start.Substring(0, indexEnd).Replace("<strong>", "").Replace(">", "").Replace("\n", "").Replace(" ", ""); if (end.Contains("<strongtitle=")) { try { end = end.Split('\"')[6].Replace("strongtitle=", ""); } catch { } } FollowerCount = rgx.StripTagsRegex(end).Replace("data-nav=\"followers\"", string.Empty).Replace("\n", string.Empty).Replace("\"", string.Empty).Trim(); } catch { } try { int indexStart1 = PageSource.IndexOf("data-element-term=\"following_stats"); string start1 = PageSource.Substring(indexStart1).Replace("data-element-term=\"following_stats", ""); int indexEnd1 = start1.IndexOf("</strong>"); string end1 = start1.Substring(0, indexEnd1).Replace("<strong>", "").Replace(">", "").Replace("\n", "").Replace(" ", ""); if (end1.Contains("<strongtitle=")) { try { end1 = end1.Split('\"')[6].Replace("strongtitle=", ""); } catch { } } FollwingCount = rgx.StripTagsRegex(end1).Replace("data-nav=\"following\"", string.Empty).Replace("\n", string.Empty).Replace("\"", string.Empty).Trim(); } catch { } } else { try { int indexStart = PageSource.IndexOf("ProfileNav-item ProfileNav-item--following"); string start = PageSource.Substring(indexStart).Replace("ProfileNav-item ProfileNav-item--following", ""); int indexEnd = start.IndexOf("data-nav=\"following"); string end = start.Substring(0, indexEnd).Replace("data-nav=\"following", "").Replace("<a class=\"ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav u-textUserColor\" title=", "").Replace("Following", "").Replace(",", "").Replace("\n", "").Replace("\"", "").Replace(">", "").Trim(); FollwingCount = rgx.StripTagsRegex(end).Replace("data-nav=\"followers\"", string.Empty).Replace("\n", string.Empty).Replace("\"", string.Empty).Replace("following", string.Empty).Trim(); } catch { } try { int indexStart = PageSource.IndexOf("ProfileNav-item ProfileNav-item--follower"); string start = PageSource.Substring(indexStart).Replace("ProfileNav-item ProfileNav-item--follower", ""); int indexEnd = start.IndexOf("data-nav=\"followers"); string end = start.Substring(1, indexEnd).Replace("data-nav=\"followers", "").Replace("<a class=\"ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav u-textUserColor\" title=", "").Replace("followers", "").Replace(",", "").Replace("\n", "").Replace("\"", "").Replace(">", "").Replace("Followers", "").Trim(); FollowerCount = rgx.StripTagsRegex(end).Replace("data-nav=\"followers\"", string.Empty).Replace("\n", string.Empty).Replace("\"", string.Empty).Replace("followers", string.Empty).Trim(); if(FollowerCount.Contains(" ")) { FollowerCount = FollowerCount.Split(' ')[0].Trim(); } } catch { } } AccountStatus = "Active"; IsNotSuspended = true; } else if (PageSource.Contains("Account suspended") || URL.Contains("https://twitter.com/account/suspended") || PageSource.Contains("account-suspended")) { Log("[ " + DateTime.Now + " ] => [ " + Username + " - Account Suspended ] "); //reminveSuspendedAccounts(Username); clsDBQueryManager database = new clsDBQueryManager(); database.UpdateSuspendedAcc(Username); AccountStatus = "Account Suspended"; IsNotSuspended = false; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(Username + ":" + Password + ":" + proxyAddress + ":" + proxyPort + ":" + proxyUsername + ":" + proxyPassword, Globals.path_SuspendedEmailAccounts); } } catch (Exception ex) { } }
public string NewPin(string Board, string Message, string BlogUrl, ref PinInterestUser objPinUser) { try { string[] arrPinId = Regex.Split(BlogUrl, "/"); foreach (var itemPinId in arrPinId) { if (PDGlobals.ValidateNumber(itemPinId)) { PinId = itemPinId; } } string getPinPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(BlogUrl), "", "", ""); if (getPinPageSource.Contains("description_html")) { description = Utils.Utils.getBetween(getPinPageSource, "description_html\":", ", \"title\":").Replace("\"", "").Replace("&", "%26").Trim(); description = description.Replace(" ", "+").Replace(",", "%2C").Replace("amp;", ""); } if (getPinPageSource.Contains("serving_link")) { link = Utils.Utils.getBetween(getPinPageSource, "serving_link\":", ", \"is_promoted").Replace("\"", "").Trim(); link = link.Replace(":", "%3A").Replace("/", "%2F").Replace("?", "%3F").Replace("=", "%3D").Replace("&", "%26"); } if (getPinPageSource.Contains("board_id")) { if (string.IsNullOrEmpty(Board)) { Random rnd = new Random(); int BoardNum = rnd.Next(0, objPinUser.lstBoardId.Count - 1); boardId = objPinUser.lstBoardId[BoardNum]; } else { boardId = Board; } GlobusLogHelper.log.Info(" => [ Repining " + boardId + " For " + objPinUser.Username + " ]"); } else { boardId = Board; } string BlogUrl1 = BlogUrl.Replace(":", "%253A").Replace("/", "%252F").Replace("?", "%3F").Replace("=", "%3D").Replace("&", "%26"); string BlogUrl2 = BlogUrl.Replace(":", "%3A").Replace("/", "%2F").Replace("?", "%3F").Replace("=", "%3D").Replace("&", "%26"); string Checking = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri("https://www.pinterest.com")); if (Checking.Contains("profileName")) { } else { ObjAccountManager.LoginPinterestAccount(ref objPinUser); } string redirectDomain = GlobusHttpHelper.valueURl.Split('.')[0]; string newHomePageUrl = redirectDomain + "." + "pinterest.com"; if (Checking.Contains("profileName")) { string RepinpostData = "source_url=%2Fpin%2Ffind%2F%3Furl%3D" + BlogUrl1 + "&data=%7B%22options%22%3A%7B%22method%22%3A%22scraped%22%2C%22description%22%3A%22" + Message.Replace(" ", "+") + "%22%2C%22link%22%3A%22" + BlogUrl2 + "%22%2C%22image_url%22%3A%22" + BlogUrl2 + "%22%2C%22board_id%22%3A%22" + Board + "%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=App%3EModalManager%3EModal%3EPinCreate%3EBoardPicker%3ESelectList(view_type%3DpinCreate%2C+selected_section_index%3Dundefined%2C+selected_item_index%3Dundefined%2C+highlight_matched_text%3Dtrue%2C+suppress_hover_events%3Dundefined%2C+scroll_selected_item_into_view%3Dtrue%2C+select_first_item_after_update%3Dfalse%2C+item_module%3D%5Bobject+Object%5D)"; string PostUrl = redirectDomain + ".pinterest.com/resource/PinResource/create/"; try { Thread.Sleep(2 * 1000); RepinPagesource = objPinUser.globusHttpHelper.postFormDataProxyPin(new Uri(PostUrl), RepinpostData, newHomePageUrl); } catch (Exception ex) { Thread.Sleep(1 * 30 * 1000); try { RepinPagesource = objPinUser.globusHttpHelper.postFormDataProxyPin(new Uri(PostUrl), RepinpostData, newHomePageUrl); } catch (Exception) { Thread.Sleep(1 * 1000); RepinPagesource = objPinUser.globusHttpHelper.postFormDataProxyPin(new Uri(PostUrl), RepinpostData, newHomePageUrl); } } if (!string.IsNullOrEmpty(RepinPagesource)) { GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " is Done. ]"); return "true"; } else { GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " is Failed. ]"); return "false"; } GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " ]"); if (getPinPageSource.Contains("class=\"sourceFlagWrapper")) { BaseLib.GlobusRegex rgx = new GlobusRegex(); string urldata = System.Text.RegularExpressions.Regex.Split(System.Text.RegularExpressions.Regex.Split(getPinPageSource, "sourceFlagWrapper")[1], "</a>")[0]; linkurl = rgx.GetHrefUrlTag(urldata).Replace("href=\"", string.Empty); } else if (string.IsNullOrEmpty(linkurl)) { try { string urldata = System.Text.RegularExpressions.Regex.Split(System.Text.RegularExpressions.Regex.Split(getPinPageSource, "sourceFlagWrapper")[1], "</a>")[0]; string Datavalue = urldata.Substring(urldata.IndexOf("href=\\\"")); int startindex = Datavalue.IndexOf("href=\\\""); string start = Datavalue.Substring(startindex).Replace("href=\\\"", ""); int endindex = start.IndexOf("\\\""); string end = start.Substring(0, endindex); linkurl = end; } catch { }; } try { string postdata1 = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + Message + "%22%2C%22link%22%3A%22" + Uri.EscapeDataString(linkurl) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + objPinUser.App_version + "%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D" + PinId + "))%3EPinActionBar(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + ")))"; string afterposting = objPinUser.globusHttpHelper.postFormDataProxy(new Uri("https://www.pinterest.com/resource/RepinResource/create/"), postdata1, "http://www.pinterest.com/pin/" + PinId + "/", "", 0, "", ""); return "true"; } catch (Exception ex) { }; } else { GlobusLogHelper.log.Info(" => [ Login Issue " + " For " + objPinUser.Username + " ]"); } } catch(Exception ex) { GlobusLogHelper.log.Error(" Error :" + ex.StackTrace); } return "false"; }
public List<StructTweetIDs> TweetExtractor_ByUserName_New(string keyword) { lst_structTweetIDs = new List<StructTweetIDs>(); string user_name = string.Empty; int i = 0; try { string HomePagedata = string.Empty; //startAgain: //if (i == 0) { //HomePagedata = globushttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/profiles/show/" + keyword + "/timeline?include_available_features=1&include_entities=1"), "", ""); //&composed_count=0&count=" + noOfRecords + "" //HomePagedata = globushttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/profiles/show/" + keyword + "/timeline?include_available_features=1&include_entities=1&composed_count=0&count=" + noOfRecords + ""), "", ""); HomePagedata = globushttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + noOfRecords + "&include_available_features=1&include_entities=1"),"",""); } //else //{ // if (HomePagedata.Contains("\"has_more_items\":true")) // { // HomePagedata = globushttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/profiles/show/" + keyword + "/timeline?include_available_features=1&include_entities=1&max_id=" + lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet), "", ""); // } // else // { // return lst_structTweetIDs; // } //} if (!string.IsNullOrEmpty(HomePagedata)) { JObject Abc = JObject.Parse(HomePagedata); string datahkj = string.Empty; foreach (object data in Abc) { datahkj = data.ToString(); } string[] splitRes = Regex.Split(datahkj, "ProfileTweet u-textBreak js-tweet js-stream-tweet js-actionable-tweet");//Regex.Split(res_Get_searchURL, "{\"created_at\""); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { string modified_Item = string.Empty; string text = string.Empty; string TweeterUserId = string.Empty; string TweeterUserScreanName = string.Empty; string Tweetid = string.Empty; ///Tweet ID try { int startindex = item.IndexOf("data-item-id=\\\""); string start = item.Substring(startindex).Replace("data-item-id=\\\"", ""); int endindex = start.IndexOf("\\\""); string end = start.Substring(0, endindex); Tweetid = end; } catch (Exception ex) { } ///Tweet User Screen name try { int startindex = item.IndexOf("data-screen-name=\\\""); string start = item.Substring(startindex).Replace("data-screen-name=\\\"", ""); int endindex = start.IndexOf("\\\""); string end = start.Substring(0, endindex); TweeterUserScreanName = end; } catch (Exception ex) { } ///Tweet User User-id try { int startindex = item.IndexOf("data-user-id=\\\""); string start = item.Substring(startindex).Replace("data-user-id=\\\"", ""); int endindex = start.IndexOf("\\\""); string end = start.Substring(0, endindex); TweeterUserId = end; } catch (Exception ex) { } ///Tweet Text try { #region code commented by PUJA //int startindex = item.IndexOf("ProfileTweet-text js-tweet-text u-dir"); //string start = item.Substring(startindex).Replace("ProfileTweet-text js-tweet-text u-dir", ""); //int endindex = start.IndexOf("</p>"); //string end = start.Substring(0, endindex); //text = end.Replace("class=\\\"invisible\\\"", "").Replace("<b", "").Replace("</b", "").Replace("<s", "").Replace("</s", "").Replace("class=\\\"twitter-atreply pretty-link\\\" dir=\\\"ltr\\\"", "").Replace(">", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"invisible\\\">", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace(" ", "").Replace("</a", "").Replace("</span", "").Replace("<span", "").Replace("<a href=", "").Replace("rel=nofollow dir=ltr data-expanded-url=", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=\\\"", "").Replace("class=\\\"twitter-timeline-link\\\" target=\\\"_blank\\\" title=\\\"", "").Replace("class=\\\"twitter-timeline-link u-isHiddenVisually\\\" data-pre-embedded=\\\"true\\\" dir=\\\"ltr\\\"","").Trim(); //text = text.Replace("<", "").Replace("\\\"", "").Replace("\\n","").Replace("\"","").Trim(); //text = text.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); //text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", "").Replace("&", "&").Replace("'", "'").Replace("<", "<").Replace(">", ">").Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("#", string.Empty).Replace("dir=ltr", "").Trim(); //text = text.Replace("\"", "").Replace("<", "").Replace("\\\"", "").Replace("\\", ""); //string[] array = Regex.Split(text, "http"); //text = string.Empty; //foreach (string itemData in array) //{ // if (!itemData.Contains("t.co")) // { // string data = string.Empty; // if (itemData.Contains("//")) // { // data = "http" + itemData; // if (!text.Contains(itemData.Replace(" ", ""))) // { // text += data; // } // } // else // { // if (!text.Contains(itemData.Replace(" ", ""))) // { // text += itemData; // } // } // } // if (text.Contains("data-aria-label-part=0")) // { // text = globushttpHelper.getBetween(text + ":&$#@", "data-aria-label-part=0", ":&$#@"); // } //} #endregion GlobusRegex regx = new GlobusRegex(); // foreach (string item1 in splitRes) string tweetUserid = string.Empty; int startindex = item.IndexOf("ProfileTweet-text js-tweet-text u-dir"); if (startindex == -1) { startindex = item.IndexOf("js-tweet-text tweet-text"); } string start = item.Substring(startindex).Replace("ProfileTweet-text js-tweet-text u-dir", "").Replace("js-tweet-text tweet-text tweet-text-rtl\\\"", ""); int endindex = start.IndexOf("</p>"); if (endindex == -1) { endindex = 0; endindex = start.IndexOf("stream-item-footer"); } string end = start.Substring(0, endindex); end = regx.StripTagsRegex(end); text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "").Replace("dir=\"ltr\"", ""); text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", "").Replace("&", "&").Replace("'", "'").Replace("<", "<").Replace(">", ">"); string[] array = Regex.Split(text, "http"); text = string.Empty; foreach (string itemData in array) { if (!itemData.Contains("t.co")) { string data = string.Empty; if (itemData.Contains("//")) { data = ("http" + itemData).Replace(" span ", string.Empty); if (!text.Contains(itemData.Replace(" ", "")))// && !data.Contains("class") && !text.Contains(data)) { text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty); } } else { if (!text.Contains(itemData.Replace(" ", ""))) { text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("\n", string.Empty).Replace(" ", string.Empty).Replace(" lang=endata-aria-label-part=0", string.Empty); } } } if (text.Contains("data-aria-label-part=0")) { text = globushttpHelper.getBetween(text + ":&$#@", "data-aria-label-part=0", ":&$#@"); } } } catch (Exception ex) { } StructTweetIDs structTweetIDs = new StructTweetIDs(); structTweetIDs.ID_Tweet = Tweetid; structTweetIDs.ID_Tweet_User = TweeterUserId; structTweetIDs.username__Tweet_User = TweeterUserScreanName; structTweetIDs.wholeTweetMessage = text; //lst_structTweetIDs.Add(structTweetIDs); if (lst_structTweetIDs.Count < noOfRecords) { lst_structTweetIDs.Add(structTweetIDs); } else { break; } } } //i++; //goto startAgain; return lst_structTweetIDs; } catch (Exception ex) { return lst_structTweetIDs; } }
//Function for Returning TweetData to Wait and reply public List<StructTweetIDs> GetTweetData_WaitReply(string keyword) { try { try { lst_structTweetIDs = new List<StructTweetIDs>(); //AddToTweetCreatorLogs("[ " + DateTime.Now + " ] => [ Extracting Tweets for " + keyword + " ]"); string[] arraylst = new string[] { }; string scroll_cursor = "0"; GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); for (int i = 0; i < noOfRecords; i++) { //AddToTweetCreatorLogs("[ " + DateTime.Now + " ] => [ Getting " + (i + 1) + " Page Tweets ]"); string pgsrcs = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&scroll_cursor=" + scroll_cursor), "", ""); //Getting the pages try { int startindex = pgsrcs.IndexOf("scroll_cursor"); string start = pgsrcs.Substring(startindex).Replace("scroll_cursor", string.Empty); int endindex = start.IndexOf("refresh_cursor"); string end = string.Empty; if (endindex >= 0) { end = start.Substring(0, endindex); scroll_cursor = end.Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); } else { endindex = start.IndexOf("\"}"); end = start.Substring(0, endindex); scroll_cursor = end; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> StartKeywordExtracting() --> Getting Maxid --> " + ex.Message, Globals.Path_TweetCreatorErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> StartKeywordExtracting() --> Getting Maxid --> " + ex.Message, Globals.Path_TwtErrorLogs); } //getting the information:tweets,username,userid,tweetid JObject Abc = JObject.Parse(pgsrcs); string datahkj = string.Empty; datahkj = Abc["items_html"].ToString(); string[] splitRes = Regex.Split(((string)Abc["items_html"]), "js-stream-item stream-item stream-item expanding-stream-item");//Regex.Split(res_Get_searchURL, "{\"created_at\""); splitRes = splitRes.Skip(1).ToArray(); GlobusRegex regx = new GlobusRegex(); foreach (string item in splitRes) { string from_user = string.Empty; string from_user_id = string.Empty; string from_user_name = string.Empty; string id = string.Empty; string text = string.Empty; ///Tweet ID try { int startindex = item.IndexOf("data-item-id=\""); string start = item.Substring(startindex).Replace("data-item-id=\"", ""); int endindex = start.IndexOf("\""); string end = start.Substring(0, endindex); id = end; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> userid --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> userid --> " + ex.Message, Globals.Path_TwtErrorLogs); } ///Tweet User Screen name try { int startindex = item.IndexOf("data-screen-name=\""); string start = item.Substring(startindex).Replace("data-screen-name=\"", ""); int endindex = start.IndexOf("\""); string end = start.Substring(0, endindex); from_user_name = end; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> from_user_name --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> from_user_name --> " + ex.Message, Globals.Path_TwtErrorLogs); } ///Tweet User User-id try { int startindex = item.IndexOf("data-user-id=\""); string start = item.Substring(startindex).Replace("data-user-id=\"", ""); int endindex = start.IndexOf("\""); string end = start.Substring(0, endindex); from_user_id = end; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } ///Tweet Text try { int startindex = item.IndexOf("js-tweet-text tweet-text\""); string start = item.Substring(startindex).Replace("js-tweet-text tweet-text\"", ""); int endindex = start.IndexOf("</p>"); string end = start.Substring(0, endindex); end = regx.StripTagsRegex(end); text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); string[] array = Regex.Split(text, "http"); text = string.Empty; foreach (string itemData in array) { if (!itemData.Contains("t.co")) { string data = string.Empty; if (itemData.Contains("//")) { data = "http" + itemData; if (!text.Contains(itemData.Replace(" ", ""))) { text += data; } } else { if (!text.Contains(itemData.Replace(" ", ""))) { text += itemData; } } } } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwtErrorLogs); } StructTweetIDs structTweetIDs = new StructTweetIDs(); structTweetIDs.ID_Tweet = id; structTweetIDs.ID_Tweet_User = from_user_id; structTweetIDs.username__Tweet_User = from_user_name; structTweetIDs.wholeTweetMessage = text; Log("[ " + DateTime.Now + " ] => [ " + id + " ]"); Log("[ " + DateTime.Now + " ] => [ " + from_user_id + " ]"); Log("[ " + DateTime.Now + " ] => [ " + from_user_name + " ]"); Log("[ " + DateTime.Now + " ] => [ " + text + " ]"); Log("---------------------------------------------------------------------------------------------------------------------------------------------------"); if (text.Contains(keyword)) { lst_structTweetIDs.Add(structTweetIDs); lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList(); } if (lst_structTweetIDs.Count == noOfRecords) { break; } } if (lst_structTweetIDs.Count == noOfRecords) { break; } } //AddToTweetCreatorLogs("[ " + DateTime.Now + " ] => [ " + lstweete.Count + " Total distinct Tweets ]"); //AddToTweetCreatorLogs("[ " + DateTime.Now + " ] => [ Finished Extracting Tweets for " + keyword + " ]"); //AddToTweetCreatorLogs("-----------------------------------------------------------------------------------------------------------------------"); } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> StartKeywordExtracting() --> " + ex.Message, Globals.Path_TweetCreatorErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> StartKeywordExtracting() --> " + ex.Message, Globals.Path_TwtErrorLogs); } return lst_structTweetIDs; } catch (Exception ex) { return lst_structTweetIDs; GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } }
public bool RepinwithMessage(string PinId, string myMessage, string Board, string NumberOfPage, ref PinInterestUser objPinUser) { try { string getPinPageSource = string.Empty; string pinUrl = string.Empty; string url = "https://www.pinterest.com/pin/" + PinId; string CheckPinPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(url), "", string.Empty, ""); if (!CheckPinPageSource.Contains("<div>Something went wrong!</div>") && !CheckPinPageSource.Contains("<div>Sorry. We've let our engineers know.</div>") && !CheckPinPageSource.Contains("<div>Whoops! We couldn't find that page.</div>") && !CheckPinPageSource.Contains("<div class=\"suggestionText\">How about these instead?</div>")) { pinUrl = "https://www.pinterest.com/pin/" + PinId + "/"; } else { GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Pin " + PinId + " Is InCorrect ]"); } try { if (!string.IsNullOrEmpty(Globals.ItemSelect)) { GlobusLogHelper.log.Info("[ " + DateTime.Now + "] => [ Repining " + PinId + " For " + objPinUser.Username + "In" + Globals.ItemSelect + " ]"); } else { GlobusLogHelper.log.Info("[ " + DateTime.Now + "] => [ Repining " + PinId + " For " + objPinUser.Username + " ]"); } } catch { }; try { GlobusHttpHelper objhttp = new GlobusHttpHelper(); getPinPageSource = objhttp.getHtmlfromUrl(new Uri(pinUrl), "", "", ""); } catch { }; string description = string.Empty; string link = string.Empty; string boardId = string.Empty; string RepinPagesource = string.Empty; string appVersion = string.Empty; try { if (getPinPageSource.Contains("description_html")) { description = Utils.Utils.getBetween(getPinPageSource, "description_html\":", ", \"title\":").Replace("\"", "").Replace("&", "%26").Trim(); description = description.Replace(" ", "+").Replace(",", "%2C").Replace("amp;", ""); } if (getPinPageSource.Contains("serving_link")) { link = Utils.Utils.getBetween(getPinPageSource, "serving_link\":", ", \"is_promoted").Replace("\"", "").Trim(); link = link.Replace(":", "%3A").Replace("/", "%2F").Replace("?", "%3F").Replace("=", "%3D").Replace("&", "%26"); } if (getPinPageSource.Contains("board_id")) { try { if (string.IsNullOrEmpty(Board)) { Random rnd = new Random(); int BoardNum = rnd.Next(0, objPinUser.Boards.Count - 1); boardId = objPinUser.Boards[BoardNum]; } else { boardId = Board; } if (string.IsNullOrEmpty(boardId.ToString())) { GlobusLogHelper.log.Info("[ " + DateTime.Now + "] => [Board is not present in your account , can't repin]"); return false; } } catch (Exception) { } } else if (getPinPageSource.Contains("board")) { boardId = Board; } lock (Lock_RepinonBoard) { string Checking = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri("https://www.pinterest.com")); if (Checking.Contains("profileName")) { if (Checking.Contains("profileName")) { appVersion = Utils.Utils.getBetween(Checking, "\"app_version\": \"", "\", \""); } } else { ObjAccountManager.LoginPinterestAccount(ref objPinUser); } string RedirectUrl = GlobusHttpHelper.valueURl.Split('.')[0]; string newHomePage = RedirectUrl + ".pinterest.com"; string linkurl = string.Empty; string RepinpostData = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22pin_id%22%3A%22" + PinId + "%22%2C%22description%22%3A%22" + myMessage + "%22%2C%22link%22%3A%22" + link + "%22%2C%22is_video%22%3Afalse%2C%22board_id%22%3A%22" + Board + "%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=Modal()%3EPinCreate3(resource%3DPinResource(id%3D" + PinId + "))%3EBoardPicker(resource%3DBoardPickerBoardsResource(filter%3Dall))%3ESelectList(view_type%3DpinCreate3%2C+selected_section_index%3Dundefined%2C+selected_item_index%3Dundefined%2C+highlight_matched_text%3Dtrue%2C+suppress_hover_events%3Dundefined%2C+item_module%3D%5Bobject+Object%5D)"; string PostPageUrl = RedirectUrl + ".pinterest.com/resource/RepinResource/create/"; try { RepinPagesource = objPinUser.globusHttpHelper.postFormDataProxyREPin(new Uri(PostPageUrl), RepinpostData, newHomePage, objPinUser.App_version); } catch (Exception ex) { } if (string.IsNullOrEmpty(RepinPagesource)) { try { if (getPinPageSource.Contains("class=\"sourceFlagWrapper")) { try { BaseLib.GlobusRegex rgx = new GlobusRegex(); string urldata = System.Text.RegularExpressions.Regex.Split(System.Text.RegularExpressions.Regex.Split(getPinPageSource, "sourceFlagWrapper")[1], "</a>")[0]; linkurl = rgx.GetHrefUrlTag(urldata).Replace("href=\"", string.Empty); } catch (Exception ex) { } } else if (string.IsNullOrEmpty(linkurl)) { try { string urldata = System.Text.RegularExpressions.Regex.Split(System.Text.RegularExpressions.Regex.Split(getPinPageSource, "sourceFlagWrapper")[1], "</a>")[0]; string Datavalue = urldata.Substring(urldata.IndexOf("href=\\\"")); int startindex = Datavalue.IndexOf("href=\\\""); string start = Datavalue.Substring(startindex).Replace("href=\\\"", ""); int endindex = start.IndexOf("\\\""); string end = start.Substring(0, endindex); linkurl = end;// Datavalue.Substring(0, Datavalue.IndexOf("\\\">")).Replace("\\", string.Empty).Replace("\n", string.Empty).Replace(">", string.Empty).Replace("href=\"", string.Empty); } catch { }; } try { string postdata1 = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + myMessage + "%22%2C%22link%22%3A%22" + Uri.EscapeDataString(linkurl) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + objPinUser.App_version + "%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D" + PinId + "))%3EPinActionBar(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + ")))"; string afterposting = objPinUser.globusHttpHelper.postFormDataProxy(new Uri("https://www.pinterest.com/resource/RepinResource/create/"), postdata1, "http://www.pinterest.com/pin/" + PinId + "/", "", 0, "", ""); if (!afterposting.Contains("<div>Uh oh! Something went wrong.")) { return true; } return false; } catch (Exception ex) { }; } catch (Exception ex) { } } if (!string.IsNullOrEmpty(RepinPagesource)) { try { if (!string.IsNullOrEmpty(Globals.ItemSelect)) { GlobusLogHelper.log.Info("[ " + DateTime.Now + "] => [ Repining " + PinId + " For " + objPinUser.Username + " In " + Globals.ItemSelect + " is Done. ]"); } else { GlobusLogHelper.log.Info("[ " + DateTime.Now + "] => [ Repining " + PinId + " For " + objPinUser.Username + " is Done. ]"); } } catch { }; return true; } else { GlobusLogHelper.log.Info("[ " + DateTime.Now + "] => [ Repining " + PinId + " For " + objPinUser.Username + " is Failed. ]"); return false; } } } catch { }; return false; } catch (Exception Ex) { return false; } }
public List<string> GetTweetData_Scrape(string keyword, int NoOfTweets, out string returnStatus) { #region Old code //GlobusRegex regx = new GlobusRegex(); //string status = string.Empty; //List<string> lst_TweetIDs = new List<string>(); //string Nextcounter = string.Empty; //int counter = 0; //try //{ // string user_id = string.Empty; // string searchURL = string.Empty; //StartAgain: // if (NumberHelper.ValidateNumber(keyword)) // { // //searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&id=" + keyword + "&count=" + TweetExtractCount; // //break; // } // else // { // searchURL = "https://twitter.com/"+keyword; // //searchURL = "https://twitter.com/i/profiles/show/" + keyword + "/timeline?include_available_features=1&include_entities=1&max_id=236830342521171967"; // ////searchURL = "https://twitter.com/i/profiles/show/sachin_rt/media_timeline?max_id=248336419444101121&oldest_unread_id=0"; // ////searchURL = "https://api.twitter.com/1/statuses/user_timeline.json?include_entities=true&include_rts=true&screen_name=" + keyword + "&count=" + TweetExtractCount; // } // ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); // string res_Get_searchURL = HttpHelper.GetHtml(searchURL); // if (res_Get_searchURL.Contains("max_id"))// && !res_Get_searchURL.Contains("Sorry, that page does not exist") && !res_Get_searchURL.Contains("Not authorized") && res_Get_searchURL.Contains("created_at") && !string.IsNullOrEmpty(res_Get_searchURL)) // { // if (res_Get_searchURL.Contains("\"has_more_items\":true")) // { // try // { // int startindex = res_Get_searchURL.IndexOf("{\"max_id\":\""); // string start = res_Get_searchURL.Substring(startindex).Replace("{\"max_id\":\"", ""); // int endindex = start.IndexOf("\","); // string end = start.Substring(0, endindex); // Nextcounter = end; // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs); // } // } // JObject Abc = JObject.Parse(res_Get_searchURL); // string datahkj = string.Empty; // foreach (object data in Abc) // { // datahkj = data.ToString(); // } // string[] splitRes = Regex.Split(datahkj, "js-stream-item stream-item stream-item expanding-stream-item"); // splitRes = splitRes.Skip(1).ToArray(); // foreach (string splitRes_item in splitRes) // { // ///Tweet Text // string text = string.Empty; // try // { // int startindex = splitRes_item.IndexOf("js-tweet-text tweet-text\\\""); // string start = splitRes_item.Substring(startindex).Replace("js-tweet-text tweet-text\\\"", ""); // int endindex = start.IndexOf("</p>"); // string end = start.Substring(0, endindex); // end = regx.StripTagsRegex(end); // text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); // text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); // string[] array = Regex.Split(text, "http"); // text = string.Empty; // foreach (string itemData in array) // { // if (!itemData.Contains("t.co")) // { // string data = string.Empty; // if (itemData.Contains("//")) // { // data = "http" + itemData; // if (!text.Contains(itemData.Replace(" ", ""))) // { // text += data; // } // } // else // { // if (!text.Contains(itemData.Replace(" ", ""))) // { // text += itemData; // } // } // } // } // //Remove RT or @ mentions // #region -----RT or @ mentions------- // if (RemoveRTMSg && text.StartsWith("RT")) // continue; // if (removeAtMentions && text.Contains("@")) // continue; // #endregion // string txtdata = keyword + ":" + (text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty)); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(txtdata, Globals.Path_TweetExtractor); // lst_TweetIDs.Add(txtdata); // status = "No Error"; // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwtErrorLogs); // } // } // if (lst_TweetIDs.Count == NoOfTweets && !string.IsNullOrEmpty(Nextcounter)) // { // goto StartAgain; // } //string[] splitRes = Regex.Split(res_Get_searchURL, "{\"created_at"); ////string[] splitRes = Regex.Split(res_Get_searchURL, "js-stream-item stream-item stream-item expanding-stream-item"); //splitRes = splitRes.Skip(1).ToArray(); #region //foreach (string item in splitRes) //{ // string Tweet = Globussoft.GlobusHttpHelper.ParseJson(item, "text"); // Regex regex = new Regex(@"\\u([0-9a-z]{4})", RegexOptions.IgnoreCase); // Tweet = regex.Replace(Tweet, match => char.ConvertFromUtf32(Int32.Parse(match.Groups[1].Value, System.Globalization.NumberStyles.HexNumber))); // //Remove RT or @ mentions // #region -----RT or @ mentions------- // if (RemoveRTMSg && Tweet.StartsWith("RT")) // continue; // if (removeAtMentions && Tweet.Contains("@")) // continue; // #endregion // string data = keyword + ":" + (Tweet.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty)); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(data, Globals.Path_TweetExtractor); // lst_TweetIDs.Add(data); //} //status = "No Error"; #endregion // } // else if (res_Get_searchURL.Contains("Rate limit exceeded")) // { // status = "Rate limit exceeded"; // } // else if (res_Get_searchURL.Contains("Sorry, that page does not exist")) // { // status = "Sorry, that page does not exist"; // } // else if (res_Get_searchURL.Contains("Not authorized")) // { // status = "Not Authorized"; // } // else if (string.IsNullOrEmpty(res_Get_searchURL)) // { // status = "Not Authorized"; // } // else // { // status = "Empty"; // } //} //catch (Exception ex) //{ // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); // status = "Error"; //} //returnStatus = status; //return lst_TweetIDs; #endregion List<string> lst_Tweets = new List<string>(); string status = string.Empty; //string Nextcounter = "0"; try { #region commented code //StartAgain: //string searchURL = "https://twitter.com/i/search/timeline?type=relevance&src=typd&include_available_features=1&include_entities=1&max_id=" + Nextcounter + "&q=" + Uri.EscapeDataString(keyword); //string searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=40&include_available_features=1&include_entities=1&include_new_items_bar=true&interval=60000&latent_count=0&since_id=" + Nextcounter + ""; //searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + NoOfTweets + "&include_available_features=1&include_entities=1&max_id=352346266648326145"; #endregion int TempNoOfTweets = NoOfTweets * 1; string searchURL = string.Empty; searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + TempNoOfTweets + "&include_available_features=1&include_entities=1"; string res_Get_searchURL = string.Empty; try { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } catch (Exception ex) { res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL); string DataHtml = string.Empty; try { DataHtml = (string)avc["items_html"]; } catch { }; if (string.IsNullOrEmpty(DataHtml)) { foreach (object data in avc) { DataHtml = data.ToString(); } } //string[] splitRes = Regex.Split(DataHtml, "js-stream-item stream-item stream-item expanding-stream-item");//Regex.Split(res_Get_searchURL, "{\"created_at\""); string[] splitRes = Regex.Split(DataHtml, "ProfileTweet u-textBreak js-tweet js-stream-tweet js-actionable-tweet"); if (splitRes.Count() == 1) { splitRes = Regex.Split(DataHtml, "js-stream-item stream-item stream-item expanding-stream-item"); } splitRes = splitRes.Skip(1).ToArray(); GlobusRegex regx = new GlobusRegex(); foreach (string item in splitRes) { string text = string.Empty; string tweetUserid = string.Empty; string tweetURL = string.Empty; ///Tweet Text try { int startindex = item.IndexOf("ProfileTweet-text js-tweet-text u-dir"); if (startindex == -1) { startindex = item.IndexOf("js-tweet-text tweet-text"); } string start = item.Substring(startindex).Replace("ProfileTweet-text js-tweet-text u-dir", "").Replace("js-tweet-text tweet-text tweet-text-rtl\\\"", ""); int endindex = start.IndexOf("</p>"); if (endindex == -1) { endindex = 0; endindex = start.IndexOf("stream-item-footer"); } string end = start.Substring(0, endindex); end = regx.StripTagsRegex(end); text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "").Replace("dir=\"ltr\"", "").Replace(" ", " "); text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", "").Replace("&", "&").Replace("'", "'").Replace("<", "<").Replace(">", ">").Replace(" ", " "); string[] array = Regex.Split(text, "http"); text = string.Empty; foreach (string itemData in array) { if (!itemData.Contains("t.co")) { string data = string.Empty; if (itemData.Contains("//")) { data = ("http" + itemData).Replace(" span ", string.Empty); if (!text.Contains(itemData.Replace(" ", "")))// && !data.Contains("class") && !text.Contains(data)) { text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty); } } else { if (!text.Contains(itemData.Replace(" ", ""))) { text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("\n", string.Empty).Replace(" ", string.Empty).Replace(" lang=endata-aria-label-part=0", string.Empty); } } } } if (text.Contains("data-aria-label-part=0")) { text = globushttpHelper.getBetween(text + ":&$#@", "data-aria-label-part=0", ":&$#@").Replace(" ", " "); } try { int startIndex = item.IndexOf("data-tweet-id=\""); string start1 = item.Substring(startIndex).Replace("data-tweet-id=\"", ""); int endIndex = start1.IndexOf("\""); string end1 = start1.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end1; } catch (Exception ex) { tweetUserid = "null"; Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs); } #region Scrap Url try { //data-permalink-path=" int startIndex = item.IndexOf("data-permalink-path=\""); string start1 = item.Substring(startIndex).Replace("data-permalink-path=\"", ""); int endIndex = start1.IndexOf("\""); string end1 = start1.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetURL = "https://twitter.com" + end1; } catch { }; #endregion //Remove RT or @ mentions #region -----RT or @ mentions------- if (RemoveRTMSg && item.Contains("Retweeted by")) continue; if (removeAtMentions && text.Contains("@")) continue; #endregion //Path_TweetExtractorCSV if (lst_Tweets.Count() < NoOfTweets) { string txtdata = keyword + ":" + tweetUserid + ":" + (text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("dir=ltr", "").Replace("lang=en data-aria-label-part=0", "").Trim()).Replace(" ", " "); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(txtdata, Globals.Path_TweetExtractor); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("dir=ltr", "").Replace("lang=en data-aria-label-part=0", "").Trim(), Globals.Path_TweetExtractorUpload); lst_Tweets.Add(txtdata); //string tweetData=text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("dir=ltr", "").Replace("lang=en data-aria-label-part=0", "").Trim(); //ScrapTweetsForTweetModule if (!File.Exists(Globals.Path_TweetExtractorCSV)) { string Header = "UserId" + "," + "Keyword" + ","+"Message"+"," + "URL"; GlobusFileHelper.AppendStringToTextfileNewLine(Header, Globals.Path_TweetExtractorCSV); } string CheckTweetExtractor = tweetUserid + "," + keyword.Replace(",", " ") + "," + text.Replace(",", " ").Replace(" ", " ") + "," + tweetURL; if (!string.IsNullOrEmpty(tweetURL)) { GlobusFileHelper.AppendStringToTextfileNewLine(CheckTweetExtractor, Globals.Path_TweetExtractorCSV); //AddToLog_Checker("[ " + DateTime.Now + " ] => [ " + userEmail + "-" + userID + "-" + ScreanName + " ]"); } } status = "No Error"; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> text --> " + ex.Message, Globals.Path_TwtErrorLogs); } } #region commented //if (res_Get_searchURL.Contains("\"has_more_items\":true")) //{ // try // { // int startindex = res_Get_searchURL.IndexOf("{\"max_id\":\""); // string start = res_Get_searchURL.Substring(startindex).Replace("{\"max_id\":\"", ""); // int endindex = start.IndexOf("\","); // string end = start.Substring(0, endindex); // Nextcounter = end; // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetTweetData_New() -- " + keyword + " --> res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetTweetData_New() -- " + keyword + " --> res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs); // } // if (lst_Tweets.Count == NoOfTweets && !string.IsNullOrEmpty(Nextcounter)) // { // goto StartAgain; // } //} #endregion } catch (Exception ex) { status = "Empty"; } returnStatus = status; return lst_Tweets; }
private void btnStart_Searching_Click(object sender, EventArgs e) { AllOfTheseWords = (txtAllofTheseKeywords.Text).ToString(); ThisExtractPhrase = (txtThisExactPhrase.Text).ToString(); AnyOfTheseWords = (txtAnyOfTheseWords.Text).ToString(); TheseHashTags = (txtTheseHashTags.Text).ToString(); NoneOfTheseWords = (txtNoneofTheseWords.Text).ToString(); FromTheseAccounts = (txtFromTheseAccounts.Text).ToString(); ToTheseAccounts = (txtToTheseAccounts.Text).ToString(); MentionTheseAccounts = (txtMentioningTheseAccounts.Text).ToString(); NearThisPlace = (txtNearThisPlace.Text).ToString(); AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => Process Started"); try { if (string.IsNullOrEmpty(ThisExtractPhrase)) { ThisExtractPhrase = ""; } else { ThisExtractPhrase = "%20%22" + ThisExtractPhrase; } } catch { } try { if (string.IsNullOrEmpty(AnyOfTheseWords)) { AnyOfTheseWords = ""; } else { AnyOfTheseWords = "%22%20" + AnyOfTheseWords; } } catch { } try { if (string.IsNullOrEmpty(TheseHashTags)) { TheseHashTags = ""; } else { TheseHashTags = "%20%23" + TheseHashTags; } } catch { } try { if (string.IsNullOrEmpty(NoneOfTheseWords)) { NoneOfTheseWords = ""; } else { NoneOfTheseWords = "%20-" + NoneOfTheseWords; } } catch { } try { if (string.IsNullOrEmpty(FromTheseAccounts)) { FromTheseAccounts = ""; } else { FromTheseAccounts = "%20from%3A" + FromTheseAccounts; } } catch { } try { if (string.IsNullOrEmpty(ToTheseAccounts)) { ToTheseAccounts = ""; } else { ToTheseAccounts = "%20to%3A" + ToTheseAccounts; } } catch { } try { if (string.IsNullOrEmpty(MentionTheseAccounts)) { MentionTheseAccounts = ""; } else { MentionTheseAccounts = "%20%40" + MentionTheseAccounts; } } catch { } try { if (string.IsNullOrEmpty(NearThisPlace)) { NearThisPlace = ""; } else { NearThisPlace = "%20near%3A%22" + NearThisPlace; } } catch { } try { if (!string.IsNullOrEmpty(txtAllofTheseKeywords.Text)) { #region Commented //try //{ // string Url = "https://twitter.com/search?f=realtime&q=" + AllOfTheseWords + ThisExtractPhrase + AnyOfTheseWords + NoneOfTheseWords + TheseHashTags + _selectedLanguage + FromTheseAccounts + ToTheseAccounts + MentionTheseAccounts + NearThisPlace + "%22%20within%3A15mi&src=typd"; // string response = _GlobusHttpHelper.getHtmlfromUrl(new Uri(Url), "", ""); //} //catch { } public List<StructTweetIDs> NewKeywordStructDataForSearchByKeyword(string keyword) #endregion { try { BaseLib.GlobusRegex regx = new GlobusRegex(); int counter = 0; string res_Get_searchURL = string.Empty; string searchURL = string.Empty; string maxid = string.Empty; string TweetId = string.Empty; string text = string.Empty; string ProfileName = string.Empty; string Location = string.Empty; string Bio = string.Empty; string website = string.Empty; string NoOfTweets = string.Empty; string Followers = string.Empty; string Followings = string.Empty; int noOfRecords = 0; try { noOfRecords = int.Parse(txtNoOfRecords.Text); } catch { } startAgain: if (counter == 0) { searchURL = "https://twitter.com/i/search/timeline?q=" + AllOfTheseWords + ThisExtractPhrase + AnyOfTheseWords + NoneOfTheseWords + TheseHashTags + _selectedLanguage + FromTheseAccounts + ToTheseAccounts + MentionTheseAccounts + NearThisPlace + "%22%20within%3A15mi&src=typd" + "&f=realtime"; counter++; } else { searchURL = "https://twitter.com/i/search/timeline?q=" + AllOfTheseWords + ThisExtractPhrase + AnyOfTheseWords + NoneOfTheseWords + TheseHashTags + _selectedLanguage + FromTheseAccounts + ToTheseAccounts + MentionTheseAccounts + NearThisPlace + "%22%20within%3A15mi&src=typd" + "&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + ""; } try { res_Get_searchURL = _GlobusHttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => Finding results for entered details "); if (string.IsNullOrEmpty(res_Get_searchURL)) { res_Get_searchURL = _GlobusHttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } try { //string sjss = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor"); //splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("refresh_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } if (item.Contains("scroll_cursor")) { int startIndex = item.IndexOf("TWEET-"); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); TweetId = end; } } } catch (Exception) { } } catch (Exception ex) { System.Threading.Thread.Sleep(2000); res_Get_searchURL = _GlobusHttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); } // && !res_Get_searchURL.Contains("has_more_items\":false") if (!string.IsNullOrEmpty(res_Get_searchURL)) { //string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); //Regex.Split(res_Get_searchURL, "\"in_reply_to_status_id_str\""); string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); splitRes = splitRes.Skip(1).ToArray(); foreach (string item in splitRes) { if (item.Contains("data-screen-name=") && !item.Contains("js-actionable-user js-profile-popup-actionable")) { //var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL); //string DataHtml = (string)avc["items_html"]; } else { continue; } string modified_Item = "\"from_user\"" + item; string id = ""; try { int startIndex = item.IndexOf("data-user-id="); string start = item.Substring(startIndex).Replace("data-user-id=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); id = end; //lst_structTweetIDs.Add(id); AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => User Id " + id); } catch (Exception ex) { id = "null"; //Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); } string from_user_id = ""; try { int startIndex = item.IndexOf("data-screen-name=\\\""); string start = item.Substring(startIndex).Replace("data-screen-name=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); from_user_id = end; AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => User ScreenName " + from_user_id); } catch (Exception ex) { from_user_id = "null"; // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper); } string tweetUserid = string.Empty; try { int startIndex = item.IndexOf("=\\\""); string start = item.Substring(startIndex).Replace("=\\\"", ""); int endIndex = start.IndexOf("\\\""); string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end; AddToLog_AdvancedSearch("[ " + DateTime.Now + " ] => Tweet Id " + tweetUserid); } catch (Exception ex) { from_user_id = "null"; } ///Tweet Text #region Commented //try //{ // int startindex = item.IndexOf("js-tweet-text tweet-text\""); // if (startindex == -1) // { // startindex = 0; // startindex = item.IndexOf("js-tweet-text tweet-text"); // } // string start = item.Substring(startindex).Replace("js-tweet-text tweet-text\"", "").Replace("js-tweet-text tweet-text tweet-text-rtl\"", ""); // int endindex = start.IndexOf("</p>"); // if (endindex == -1) // { // endindex = 0; // endindex = start.IndexOf("stream-item-footer"); // } // string end = start.Substring(0, endindex); // end = regx.StripTagsRegex(end); // text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); // text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); // string[] array = Regex.Split(text, "http"); // text = string.Empty; // foreach (string itemData in array) // { // if (!itemData.Contains("t.co")) // { // string data = string.Empty; // if (itemData.Contains("//")) // { // data = ("http" + itemData).Replace(" span ", string.Empty); // if (!text.Contains(itemData.Replace(" ", "")))// && !data.Contains("class") && !text.Contains(data)) // { // text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty); // } // } // else // { // if (!text.Contains(itemData.Replace(" ", ""))) // { // text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("js-tweet-text tweet-text", ""); // } // } // } // } //} //catch { }; #endregion twtboardpro.TwitterDataScrapper.StructTweetIDs structTweetIDs = new twtboardpro.TwitterDataScrapper.StructTweetIDs(); if (id != "null") { structTweetIDs.ID_Tweet = tweetUserid; structTweetIDs.ID_Tweet_User = id; structTweetIDs.username__Tweet_User = from_user_id; structTweetIDs.wholeTweetMessage = text; lst_structTweetIDs.Add(structTweetIDs); } //if (!File.Exists(Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv")) //{ // GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS", Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"); //} { ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr(); GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + from_user_id), "", ""); string Responce = ProfilePageSource; #region Convert HTML to XML string xHtml = objChilkat.ConvertHtmlToXml(Responce); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion int counterdata = 0; xBeginSearchAfter = null; string dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterdata == 0) { ProfileName = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else if (counterdata == 1) { website = xNode.AccumulateTagContent("text", "script|style"); counterdata++; } else { break; } xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor"); } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim(); break; } xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; Location = xNode.AccumulateTagContent("text", "script|style"); break; } int counterData = 0; xBeginSearchAfter = null; dataDescription = string.Empty; xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//location profile-field"); while ((xNode != null)) { xBeginSearchAfter = xNode; if (counterData == 0) { // NoOfTweets = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileNav-value"); NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty); counterData++; } else if (counterData == 1) { Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty); counterData++; } else if (counterData == 2) { Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty); counterData++; } else { break; } //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav"); xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor"); } if (!string.IsNullOrEmpty(from_user_id) && tweetUserid != "null") { string Id_user = tweetUserid.Replace("}]", string.Empty).Trim(); Globals.lstScrapedUserIDs.Add(Id_user); // GlobusFileHelper.AppendStringToTextfileNewLine(id + "," + from_user_id + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"); // Log("[ " + DateTime.Now + " ] => [ " + from_user_id + "," + Id_user + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]"); } } lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList(); if (lst_structTweetIDs.Count >= noOfRecords) { // return lst_structTweetIDs; } } if (lst_structTweetIDs.Count <= noOfRecords) { maxid = lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet; if (res_Get_searchURL.Contains("has_moreitems\":false")) { } else { goto startAgain; } } else { if (res_Get_searchURL.Contains("has_more_items\":false")) { } else goto startAgain; } } } catch (Exception ex) { } } } } catch { } }
//public List<string> GetRetweetData_Scrape(string keyword, out string returnStatus) //{ // string status = string.Empty; // List<string> lst_ReTweetIDs = new List<string>(); // try // { // string searchURL = string.Empty; // if (!NumberHelper.ValidateNumber(keyword)) // { // searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + RetweetExtractcount + "&include_available_features=1&include_entities=1"; // //searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?screen_name=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true"; // //searchURL = "https://api.twitter.com/1.1/statuses/retweeted_by_user.json?screen_name=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true"; // } // else if (NumberHelper.ValidateNumber(keyword)) // { // searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?id=" + keyword + "&count=" + RetweetExtractcount + "&include_entities=true"; // } // //string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); // ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); // string res_Get_searchURL = HttpHelper.GetHtml(searchURL); // // if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && res_Get_searchURL.Contains("created_at") && !string.IsNullOrEmpty(res_Get_searchURL)) // if (!res_Get_searchURL.Contains("Rate limit exceeded") && !res_Get_searchURL.Contains("Sorry, that page does not exist") && !string.IsNullOrEmpty(res_Get_searchURL)) // { // string[] arrGetRetweeted = Regex.Split(res_Get_searchURL, "js-tweet-text tweet-text"); // foreach (string item in arrGetRetweeted) // { // try // { // if (item.Contains("Retweeted by")) // { // int Startindex = item.IndexOf("\\\""); // string start = item.Substring(Startindex); // int EndIndex = start.IndexOf("\\/p"); // string End = start.Substring(0, EndIndex).Replace("\\\"", "").Replace("\\u003", "").Replace("&", ""); // End = End.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "").Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("\\", "").Replace("title=", ""); ; // } // } // // catch { } // //} // //string[] splitRes = Regex.Split(res_Get_searchURL, "status");//Regex.Split(res_Get_searchURL, "{\"created_at\""); // //splitRes = splitRes.Skip(1).ToArray(); // //foreach (string item in splitRes) // //{ // // string Tweet = string.Empty; // // string Tweeter = string.Empty; // // try // // { // // int startIndex = item.IndexOf("<text>"); // // string start = item.Substring(startIndex); // // int endIndex = start.IndexOf("</text>"); // // string end = start.Substring(0, endIndex); // // Tweet = end.Replace("<text>", ""); // // int startOfInndex = Tweet.IndexOf(":"); // // Tweeter = Tweet.Substring(0, startOfInndex); // // Tweet = Tweet.Replace(Tweeter, ""); // // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); // } // //string data = keyword + ":" + Tweeter + ":" + Tweet.Replace(":", "^"); // //Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(data, Globals.Path_RETweetExtractor); // //lst_ReTweetIDs.Add(data); // } // } // else if (res_Get_searchURL.Contains("Rate limit exceeded")) // { // status = "Rate limit exceeded"; // } // else if (res_Get_searchURL.Contains("Sorry, that page does not exist")) // { // status = "Sorry, that page does not exist"; // } // else if (res_Get_searchURL.Contains("Not authorized")) // { // status = "Not Authorized"; // } // else if (string.IsNullOrEmpty(res_Get_searchURL)) // { // status = "Not Authorized"; // } // else // { // status = "Empty"; // } // } // catch (Exception ex) // { // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); // Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); // status = "Error"; // } // returnStatus = status; // return lst_ReTweetIDs; //} #endregion public List<string> GetRetweetData_Scrape(string keyword, out string returnStatus) { string status = string.Empty; List<string> lst_ReTweetIDs = new List<string>(); try { string searchURL = string.Empty; int extendendCount = 0; extendendCount = RetweetExtractcount * 10; if (!NumberHelper.ValidateNumber(keyword)) { searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + extendendCount + "&include_available_features=1&include_entities=1"; } else if (NumberHelper.ValidateNumber(keyword)) { searchURL = "https://api.twitter.com/1/statuses/retweeted_by_user.xml?id=" + keyword + "&count=" + extendendCount + "&include_entities=true"; } //string res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", ""); ChilkatHttpHelpr HttpHelper = new ChilkatHttpHelpr(); string res_Get_searchURL = HttpHelper.GetHtml(searchURL); if (string.IsNullOrEmpty(res_Get_searchURL)) { res_Get_searchURL = HttpHelper.GetHtml(searchURL); } var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL); string DataHtml = string.Empty; try { DataHtml = (string)avc["items_html"]; } catch { }; if (string.IsNullOrEmpty(DataHtml)) { foreach (object data in avc) { DataHtml = data.ToString(); } } string[] splitRes = Regex.Split(DataHtml, "ProfileTweet u-textBreak js-tweet js-stream-tweet js-actionable-tweet");//Regex.Split(res_Get_searchURL, "{\"created_at\""); if (splitRes.Count() == 1) { splitRes = Regex.Split(DataHtml, "ProfileTweet-action ProfileTweet-action--more js-more-ProfileTweet-actions"); } splitRes = splitRes.Skip(1).ToArray(); GlobusRegex regx = new GlobusRegex(); foreach (string item in splitRes) { string text = string.Empty; string tweetURL = string.Empty; string tweetUserid = string.Empty; if (item.Contains("Icon--retweeted"))//retweeted Icon { try { //int startindex = item.IndexOf("ProfileTweet-text js-tweet-text u-dir"); //string start = item.Substring(startindex).Replace("ProfileTweet-text js-tweet-text u-dir", "").Replace("js-tweet-text tweet-text tweet-text-rtl\\\"", ""); //int endindex = start.IndexOf("</p>"); //if (endindex == -1) //{ // endindex = 0; // endindex = start.IndexOf("stream-item-footer"); //} //string end = start.Substring(0, endindex); //end = regx.StripTagsRegex(end); #region sonu edited code try { string[] getText = Regex.Split(item, "TweetTextSize TweetTextSize--16px js-tweet-text tweet-text"); try { text = Utils.getBetween(getText[1], ">", "<"); } catch { }; }catch{}; #endregion //text = end.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", ""); text = text.Replace(" ", "").Replace("a href=", "").Replace("/a", "").Replace("<span", "").Replace("</span", "").Replace("class=\\\"js-display-url\\\"", "").Replace("class=\\\"tco-ellipsis\\\"", "").Replace("class=\\\"invisible\\\"", "").Replace("<strong>", "").Replace("target=\\\"_blank\\\"", "").Replace("class=\\\"twitter-timeline-link\\\"", "").Replace("</strong>", "").Replace("rel=\\\"nofollow\\\" dir=\\\"ltr\\\" data-expanded-url=", "").Replace(" "," "); text = text.Replace(""", "").Replace("<", "").Replace(">", "").Replace("\"", "").Replace("'", "'").Replace("&", "&").Replace("=>", "=>").Replace(" ", " ").Replace("\\", "").Replace("title=", "").Replace("js-tweet-text tweet-text", "").Replace(" ", " "); string[] array = Regex.Split(text, "http"); text = string.Empty; foreach (string itemData in array) { if (!itemData.Contains("t.co")) { string data = string.Empty; if (itemData.Contains("//")) { data = ("http" + itemData).Replace(" span ", string.Empty); if (!text.Contains(itemData.Replace(" ", ""))) { text += data.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("lang=en", string.Empty).Replace("dir=ltr data-aria-label-part=0", string.Empty); } } else { if (!text.Contains(itemData.Replace(" ", ""))) { text += itemData.Replace("u003c", string.Empty).Replace("u003e", string.Empty).Replace("lang=en", string.Empty).Replace("dir=ltr data-aria-label-part=0", string.Empty).Replace("\n",string.Empty).Replace(" ",string.Empty); } } } } if (text.Contains("data-aria-label-part=0")) { text = globushttpHelper.getBetween(text + ":&$#@", "data-aria-label-part=0", ":&$#@").Replace(" "," "); } //data-permalink-path=" #region Scrap try { //data-permalink-path=" int startIndex = item.IndexOf("data-tweet-id=\""); string start1 = item.Substring(startIndex).Replace("data-tweet-id=\"", ""); int endIndex = start1.IndexOf("\""); string end1 = start1.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetUserid = end1; } catch { }; #endregion #region Scrap Url try { //data-permalink-path=" int startIndex = item.IndexOf("data-permalink-path=\""); string start1 = item.Substring(startIndex).Replace("data-permalink-path=\"", ""); int endIndex = start1.IndexOf("\""); string end1 = start1.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", ""); tweetURL = "https://twitter.com" + end1; } catch { }; #endregion //Remove RT or @ mentions #region -----RT or @ mentions------- if (RemoveRTMSg && text.StartsWith("RT")) continue; if (removeAtMentions && text.Contains("@")) continue; #endregion if (lst_ReTweetIDs.Count() < RetweetExtractcount) { string txtdata = keyword + ":" + (text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("#", string.Empty)).Replace("dir=ltr", "").Replace(" ", " "); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(txtdata, Globals.Path_RETweetExtractor); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(text.Replace("\n", string.Empty).Replace("..", string.Empty).Replace("\n \"", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("js-tweet-text tweet-text", string.Empty).Replace("#", string.Empty).Replace("dir=ltr", "").Trim(), Globals.Path_RETweetExtractorUpload); lst_ReTweetIDs.Add(txtdata); //generate CSV file if (!File.Exists(Globals.Path_RETweetExtractorCSV)) { string Header = "UserId" + "," + "Keyword" + "," + "Message" + "," + "URL"; GlobusFileHelper.AppendStringToTextfileNewLine(Header, Globals.Path_RETweetExtractorCSV); } string CheckReTweetExtractor = tweetUserid + "," + keyword + "," + text.Replace(","," ") + "," + tweetURL; if (!string.IsNullOrEmpty(tweetURL)) { GlobusFileHelper.AppendStringToTextfileNewLine(CheckReTweetExtractor, Globals.Path_RETweetExtractorCSV); //AddToLog_Checker("[ " + DateTime.Now + " ] => [ " + userEmail + "-" + userID + "-" + ScreanName + " ]"); } } status = "No Error"; } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetRetweetData_Scrape() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs); } } } } catch (Exception ex) { status = "Empty"; } returnStatus = status; return lst_ReTweetIDs; }
public List<string> GetAllBoardNames_new1(string screenName,ref PinInterestUser objPinUser) { BaseLib.GlobusRegex rgx = new GlobusRegex(); Globussoft.GlobusHttpHelper httpHelper = new Globussoft.GlobusHttpHelper(); List<string> lstBoardIdNew = new List<string>(); //GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Getting All Board Names ]"); string UserUrl = "http://pinterest.com/" + screenName; string BoardPage = httpHelper.getHtmlfromUrl(new Uri(UserUrl), "", "", ""); string[] data = Regex.Split(BoardPage, "is_collaborative"); foreach (var itemdata in data) { try { string boardUrl = getBetween(itemdata, ", \"url\":", ",").Replace("\"", "").Trim(); if (!lstBoardUrls.Contains(boardUrl) && !string.IsNullOrEmpty(boardUrl)) { lstBoardUrls.Add(boardUrl); } if (itemdata.Contains("board_id")) { string boardId = getBetween(itemdata, "board_id\":", ",").Replace("\"", "").Trim(); if (!lstBoardIdNew.Contains(boardId)) { lstBoardIdNew.Add(boardId); } objPinUser.lstBoardId = lstBoardIdNew; } } catch (Exception ex) { GlobusLogHelper.log.Error("Error :" + ex.StackTrace); } } string[] Items = Regex.Split(BoardPage, "item"); int counter = 0; foreach (string item in Items) { try { if (item.Contains("id=\\\"Board") && item.Contains("boardLinkWrapper")) { //if (counter == 1) { string[] Data = System.Text.RegularExpressions.Regex.Split(item, "boardLinkWrapper"); foreach (string Dataitem in Data) { if (Dataitem.Contains("boardName")) { string BoardUrl = string.Empty; int startIndex = Dataitem.IndexOf("title"); int LastPoint = Dataitem.IndexOf("<h2"); //string Board = Dataitem.Substring(startIndex, LastPoint).Replace("\\n", string.Empty).Replace("\"", "").Replace("<div class=\\b", string.Empty).Replace(" ", string.Empty).Replace("\"title\"", "").Replace("</div", ""); string Board = Utils.Utils.getBetween(Dataitem, "<h2 class=\\\"title\\\">", "</h2>"); //BoardUrl = rgx.StripTagsRegex(Board); //try //{ // Board = getBetween(BoardUrl, ">>", "<"); // //modified done // if (Board == "") // { // Board = getBetween(BoardUrl, "title=", ">").Replace("\\", "").Trim(); // } //} //catch(Exception ex) //{ }; if (!BoardNames.Contains(Board)) { BoardNames.Add(Board); //DropDowenBox.Items.Add(Board); } } } } counter++; } } catch (Exception ex) { GlobusLogHelper.log.Error("Error :" + ex.StackTrace); } } return BoardNames; }
//public bool RepinwithMessage(string PinId, string myMessage, string Board, string NumberOfPage, ref PinInterestUser objPinUser) //{ // try // { // string CsrfMiddleToken = string.Empty; // string pinUrl = string.Empty; // string url = "https://www.pinterest.com/pin/" + PinId; // string CheckPinPageSource123 = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri("https://www.pinterest.com")); // string CheckPinPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(url), "", string.Empty, ""); // if (!CheckPinPageSource.Contains("<div>Something went wrong!</div>") && !CheckPinPageSource.Contains("<div>Sorry. We've let our engineers know.</div>") && !CheckPinPageSource.Contains("<div>Whoops! We couldn't find that page.</div>") && !CheckPinPageSource.Contains("<div class=\"suggestionText\">How about these instead?</div>")) // { // // lstRePinPin.Add(lstRepinUrl_item); // pinUrl = "https://www.pinterest.com/pin/" + PinId + "/"; // } // else // { // GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Pin " + PinId + " Is InCorrect ]"); // } // try // { // if (!string.IsNullOrEmpty(Globals.ItemSelect)) // { // GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + "In" + Globals.ItemSelect + " ]"); // } // else // { // GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " In " + BoardName + "]"); // } // } // catch { }; // try // { // getPinPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(pinUrl), "", "", ""); // } // catch { }; // try // { // if (getPinPageSource.Contains("description_html")) // { // description = Utils.Utils.getBetween(getPinPageSource, "description_html\":", ", \"title\":").Replace("\"", "").Replace("&", "%26").Trim(); // description = description.Replace(" ", "+").Replace(",", "%2C").Replace("amp;", ""); // } // if (getPinPageSource.Contains("serving_link")) // { // link = Utils.Utils.getBetween(getPinPageSource, "serving_link\":", ", \"is_promoted").Replace("\"", "").Trim(); // link = link.Replace(":", "%3A").Replace("/", "%2F").Replace("?", "%3F").Replace("=", "%3D").Replace("&", "%26"); // } // if (getPinPageSource.Contains("board_id")) // { // try // { // if (string.IsNullOrEmpty(Board)) // { // Random rnd = new Random(); // int BoardNum = rnd.Next(0, objPinUser.Boards.Count - 1); // boardId = objPinUser.Boards[BoardNum]; // //objPinUser.Boards // } // else // { // boardId = Board; // } // if (string.IsNullOrEmpty(boardId.ToString())) // { // GlobusLogHelper.log.Info(" => [Board is not present in your account , can't repin]"); // return false; // } // // Log("[ " + DateTime.Now + " ] => [ Repining " + boardId + " For " + pinterestAccountManager.Username + " ]"); // } // catch (Exception) // { }; // } // //objPinUser.globusHttpHelper = new GlobusHttpHelper(); // lock (Lock_RepinonBoard) // { // string Checking = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri("https://www.pinterest.com")); // CsrfMiddleToken = Utils.Utils.getBetween(Checking, "\"csrftoken\": \"", "\", \""); // if (Checking.Contains("profileName")) // { // } // else // { // ObjAccountManager.LoginPinterestAccount(ref objPinUser); // } // string linkurl = string.Empty; // string RepinpostData = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22pin_id%22%3A%22" + PinId + "%22%2C%22description%22%3A%22" + myMessage + "%22%2C%22link%22%3A%22" + link + "%22%2C%22is_video%22%3Afalse%2C%22board_id%22%3A%22" + boardId + "%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=Modal()%3EPinCreate3(resource%3DPinResource(id%3D" + PinId + "))%3EBoardPicker(resource%3DBoardPickerBoardsResource(filter%3Dall))%3ESelectList(view_type%3DpinCreate3%2C+selected_section_index%3Dundefined%2C+selected_item_index%3Dundefined%2C+highlight_matched_text%3Dtrue%2C+suppress_hover_events%3Dundefined%2C+item_module%3D%5Bobject+Object%5D)"; // RepinpostData = "source_url=%2F" + objPinUser.ScreenName + "%2Fpins%2F&data=%7B%22options%22%3A%7B%22pin_id%22%3A%22" + PinId + "%22%2C%22description%22%3A%22" + myMessage + "%22%2C%22link%22%3A%22" + link + "%22%2C%22is_video%22%3Afalse%2C%22board_id%22%3A%22" + boardId + "%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=App%3EModalManager%3EModal%3EPinCreate%3EBoardPicker%3ESelectList(view_type%3DpinCreate%2C+selected_section_index%3Dundefined%2C+selected_item_index%3Dundefined%2C+highlight_matched_text%3Dtrue%2C+suppress_hover_events%3Dundefined%2C+scroll_selected_item_into_view%3Dtrue%2C+select_first_item_after_update%3Dfalse%2C+item_module%3D%5Bobject+Object%5D)"; // try // { // RepinPagesource = objPinUser.globusHttpHelper.postFormDataProxyPincsrftoken(new Uri("https://www.pinterest.com/resource/RepinResource/create/"), RepinpostData, "https://www.pinterest.com/", CsrfMiddleToken); // } // catch (Exception ex) // { // } // if (string.IsNullOrEmpty(RepinPagesource)) // { // try // { // if (getPinPageSource.Contains("class=\"sourceFlagWrapper")) // { // try // { // BaseLib.GlobusRegex rgx = new GlobusRegex(); // string urldata = System.Text.RegularExpressions.Regex.Split(System.Text.RegularExpressions.Regex.Split(getPinPageSource, "sourceFlagWrapper")[1], "</a>")[0]; // linkurl = rgx.GetHrefUrlTag(urldata).Replace("href=\"", string.Empty); // } // catch (Exception ex) // { // } // } // else if (string.IsNullOrEmpty(linkurl)) // { // try // { // string urldata = System.Text.RegularExpressions.Regex.Split(System.Text.RegularExpressions.Regex.Split(getPinPageSource, "sourceFlagWrapper")[1], "</a>")[0]; // string Datavalue = urldata.Substring(urldata.IndexOf("href=\\\"")); // int startindex = Datavalue.IndexOf("href=\\\""); // string start = Datavalue.Substring(startindex).Replace("href=\\\"", ""); // int endindex = start.IndexOf("\\\""); // string end = start.Substring(0, endindex); // linkurl = end;// Datavalue.Substring(0, Datavalue.IndexOf("\\\">")).Replace("\\", string.Empty).Replace("\n", string.Empty).Replace(">", string.Empty).Replace("href=\"", string.Empty); // } // catch { }; // } // try // { // //objPinUser.globusHttpHelper.gCookies = new System.Net.CookieCollection(); // //string CookieList = "_b=\"ARdBmc3sRRBC1b7mWyRRCA3QBovSmWHWonr/rL5WVoV3ABT+imiaVF1UnHkaoTAnSzE=\"; sessionFunnelEventLogged=1; __utmt=1; logged_out=True; fba=True; csrftoken=w4YapCH4AAPIlfxhvZR3Mb9XIXVYzMXA; _pinterest_sess=\"TWc9PSZlZU1LQU1EL1pIY3UxbjlhWGYrUGtMSkRwYzc5TFFJS3d4dTU0VEthUVlHK1VhT2NOQkQzcUtkekYzaWNYN212U2lyRlRxd2dYejhoSldadGw5M0ZzK2NyS3Q5ODhSemhUMG5HVlVVOEx6R1RWdkh0aWdFalNuSkxJZmo0ZE9jMG44b2RXZzZOeU83ZVdJWVVNdWZKZFRhOWEwK0ZDckc0NVIwdi8xdytjcDBtVHZtTjZOUUdtQk5acko0ODVaakEwS3dEN0taU2NrQkE3YlZMU3I2Q0VicDRoWTl3RHJpaDVzUnJ3YmlLNmhYbFpMbFp6K2FKTlIyZUNWUDRKN3I3dFNMejQyUVRhb0JtMTk0ajBtcVhUVzlrRVlVQzZwTjRwY3UzdG1tWFpxQ3FGV0ZOeXBNV0FMeURUL0crMmZvVzVNWWlPbjZyUnh2UXN3R3pOT2c2RVNoZFVxZG9lUnlDaHJpS0dORFoydHhZMFgxR0hWWE02OUZDQmdlSkhzRDdjREszcjdvWDBTaFJXM1BFM01VUnRpYjB4U3l6alQ2eWxmQTdrWVdYcWFpOFhETDRWSGlQazR0L2xUNTdiUGdhbG9WUUtLZHdrSVFMN1VaWHk2LytNeWkzcUJJbStUejVMYkljczY1Sk5GQT0mblE0ZXFxTDVyeGk5NVJ2Mmp1SEZVakNxcGNjPQ==\"; c_dpr=1; cm_sub=none; __utma=229774877.1623437774.1450182939.1450182939.1450249426.2; __utmb=229774877.15.9.1450261114039; __utmc=229774877; __utmz=229774877.1450182939.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)"; // //string[] CookieList1234 = Regex.Split(CookieList, ";"); // //foreach (string str in CookieList1234) // //{ // // string Cokkiename = Regex.Split(str, "=")[0].Replace(" ",""); // // string CokkieValue = Regex.Split(str, "=")[1].Replace(" ", ""); // // System.Net.Cookie cookie = new System.Net.Cookie(); // // cookie.Name = Cokkiename; // // cookie.Value = CokkieValue; // // cookie.Domain = "pinterest.com"; // // objPinUser.globusHttpHelper.gCookies.Add(cookie); // //} // string postdata1 = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + myMessage + "%22%2C%22link%22%3A%22" + Uri.EscapeDataString(linkurl) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + objPinUser.App_version + "%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D" + PinId + "))%3EPinActionBar(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + ")))"; // // postdata1 = "source_url=%2F" + objPinUser.ScreenName + "%2Fpins%2F&data=%7B%22options%22%3A%7B%22pin_id%22%3A%22" + PinId + "%22%2C%22description%22%3A%22" + myMessage + "%22%2C%22link%22%3A%22" + link + "%22%2C%22is_video%22%3Afalse%2C%22board_id%22%3A%22" + Board + "%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=App%3EModalManager%3EModal%3EPinCreate%3EBoardPicker%3ESelectList(view_type%3DpinCreate%2C+selected_section_index%3Dundefined%2C+selected_item_index%3Dundefined%2C+highlight_matched_text%3Dtrue%2C+suppress_hover_events%3Dundefined%2C+scroll_selected_item_into_view%3Dtrue%2C+select_first_item_after_update%3Dfalse%2C+item_module%3D%5Bobject+Object%5D)"; // // "source_url=%2FPrashant123456%2Fdeveloper%2F&data=%7B%22options%22%3A%7B%22pin_id%22%3A%22350295677243761444%22%2C%22description%22%3A%22%22%2C%22link%22%3Anull%2C%22is_video%22%3Afalse%2C%22board_id%22%3A%22350295745955657898%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=App%3EModalManager%3EModal%3EPinCreate%3EBoardPicker%3ESelectList(view_type%3DpinCreate%2C+selected_section_index%3Dundefined%2C+selected_item_index%3Dundefined%2C+highlight_matched_text%3Dtrue%2C+suppress_hover_events%3Dundefined%2C+scroll_selected_item_into_view%3Dtrue%2C+select_first_item_after_update%3Dfalse%2C+item_module%3D%5Bobject+Object%5D)"; // postdata1 = "source_url=%2FPrashant123456%2Flovely2%2F&data=%7B%22options%22%3A%7B%22pin_id%22%3A%22350295677245529486%22%2C%22description%22%3A%22Hii%22%2C%22link%22%3A%22http%3A%2F%2Fwebsta.me%2Fp%2F945717296717057301_1591089306%22%2C%22is_video%22%3Afalse%2C%22board_id%22%3A%22350295745955657875%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=App%3EModalManager%3EModal%3EPinCreate%3EBoardPicker%3ESelectList(view_type%3DpinCreate%2C+selected_section_index%3Dundefined%2C+selected_item_index%3Dundefined%2C+highlight_matched_text%3Dtrue%2C+suppress_hover_events%3Dundefined%2C+scroll_selected_item_into_view%3Dtrue%2C+select_first_item_after_update%3Dfalse%2C+item_module%3D%5Bobject+Object%5D)"; // string afterposting = objPinUser.globusHttpHelper.postFormDataProxywithCSRFToken(new Uri("https://www.pinterest.com/resource/RepinResource/create/"), postdata1, "http://www.pinterest.com/", "", 0, "", "", "CsrfMiddleToken"); // //string afterposting = objPinUser.globusHttpHelper.postFormDataProxy(new Uri("https://www.pinterest.com/resource/RepinResource/create/"), postdata1, "http://www.pinterest.com/pin/" + PinId + "/", "", 0, "", "", CsrfMiddleToken); // if (!afterposting.Contains("<div>Uh oh! Something went wrong.")) // { // return true; // } // else // { // return false; // } // } // catch (Exception ex) // { // }; // } // catch (Exception ex) // { // } // } // if (!string.IsNullOrEmpty(RepinPagesource)) // { // try // { // if (!string.IsNullOrEmpty(Globals.ItemSelect)) // { // GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " In " + Globals.ItemSelect + " is Done. ]"); // } // else // { // GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " is Done. ]"); // } // //string RepinDetails =pinterestAccountManager.Username + "," + PinId.Replace(",","") + "," + boardId.Replace(",","") + "," + myMessage.Replace(",",""); // //GlobusFileHelper.AppendStringToTextfileNewLine(RepinDetails, ApplicationData.path_Repin); // } // catch { }; // return true; // } // else // { // GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " is Failed. ]"); // return false; // } // } // } // catch { }; // return false; // } // catch (Exception Ex) // { // return false; // } //} #region NEW_REPIN_WITH_MESSAGE public bool RepinwithMessage(string PinId, string myMessage, string Board, string NumberOfPage, ref PinInterestUser objPinUser) { try { string getPinPageSource = string.Empty; string pinUrl = string.Empty; string url = "https://www.pinterest.com/pin/" + PinId; string CheckPinPageSource = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri(url), "", string.Empty, ""); if (!CheckPinPageSource.Contains("<div>Something went wrong!</div>") && !CheckPinPageSource.Contains("<div>Sorry. We've let our engineers know.</div>") && !CheckPinPageSource.Contains("<div>Whoops! We couldn't find that page.</div>") && !CheckPinPageSource.Contains("<div class=\"suggestionText\">How about these instead?</div>")) { // lstRePinPin.Add(lstRepinUrl_item); pinUrl = "https://www.pinterest.com/pin/" + PinId + "/"; } else { GlobusLogHelper.log.Info("[ " + DateTime.Now + " ] => [ Pin " + PinId + " Is InCorrect ]"); } // Thread.Sleep(10 * 1000); //pinUrl = "https://www.pinterest.com/pin/" + PinId + "/"; //string checklogin = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri("https://www.pinterest.com")); try { if (!string.IsNullOrEmpty(Globals.ItemSelect)) { GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + "In" + Globals.ItemSelect + " ]"); } else { GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " ]"); } } catch { }; try { GlobusHttpHelper objhttp = new GlobusHttpHelper(); getPinPageSource = objhttp.getHtmlfromUrl(new Uri(pinUrl), "", "", ""); } catch { }; string description = string.Empty; string link = string.Empty; string boardId = string.Empty; string RepinPagesource = string.Empty; string appVersion = string.Empty; try { if (getPinPageSource.Contains("description_html")) { description = Utils.Utils.getBetween(getPinPageSource, "description_html\":", ", \"title\":").Replace("\"", "").Replace("&", "%26").Trim(); description = description.Replace(" ", "+").Replace(",", "%2C").Replace("amp;", ""); } if (getPinPageSource.Contains("serving_link")) { link = Utils.Utils.getBetween(getPinPageSource, "serving_link\":", ", \"is_promoted").Replace("\"", "").Trim(); link = link.Replace(":", "%3A").Replace("/", "%2F").Replace("?", "%3F").Replace("=", "%3D").Replace("&", "%26"); } if (getPinPageSource.Contains("board_id")) { try { if (string.IsNullOrEmpty(Board)) { Random rnd = new Random(); int BoardNum = rnd.Next(0, objPinUser.Boards.Count - 1); boardId = objPinUser.Boards[BoardNum]; } else { boardId = Board; } if (string.IsNullOrEmpty(boardId.ToString())) { GlobusLogHelper.log.Info(" => [Board is not present in your account , can't repin]"); return false; } // Log("[ " + DateTime.Now + " ] => [ Repining " + boardId + " For " + pinterestAccountManager.Username + " ]"); } catch (Exception) { } } else if (getPinPageSource.Contains("board")) { boardId = Board; } lock (Lock_RepinonBoard) { string Checking = objPinUser.globusHttpHelper.getHtmlfromUrl(new Uri("https://www.pinterest.com")); if (Checking.Contains("profileName")) { if (Checking.Contains("profileName")) { appVersion = Utils.Utils.getBetween(Checking, "\"app_version\": \"", "\", \""); } } else { ObjAccountManager.LoginPinterestAccount(ref objPinUser); } string RedirectUrl = GlobusHttpHelper.valueURl.Split('.')[0]; string newHomePage = RedirectUrl + ".pinterest.com"; string linkurl = string.Empty; string RepinpostData = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22pin_id%22%3A%22" + PinId + "%22%2C%22description%22%3A%22" + myMessage + "%22%2C%22link%22%3A%22" + link + "%22%2C%22is_video%22%3Afalse%2C%22board_id%22%3A%22" + Board + "%22%7D%2C%22context%22%3A%7B%7D%7D&module_path=Modal()%3EPinCreate3(resource%3DPinResource(id%3D" + PinId + "))%3EBoardPicker(resource%3DBoardPickerBoardsResource(filter%3Dall))%3ESelectList(view_type%3DpinCreate3%2C+selected_section_index%3Dundefined%2C+selected_item_index%3Dundefined%2C+highlight_matched_text%3Dtrue%2C+suppress_hover_events%3Dundefined%2C+item_module%3D%5Bobject+Object%5D)"; string PostPageUrl = RedirectUrl + ".pinterest.com/resource/RepinResource/create/"; try { RepinPagesource = objPinUser.globusHttpHelper.postFormDataProxyREPin(new Uri(PostPageUrl), RepinpostData, newHomePage, objPinUser.App_version); } catch (Exception ex) { } if (string.IsNullOrEmpty(RepinPagesource)) { try { if (getPinPageSource.Contains("class=\"sourceFlagWrapper")) { try { BaseLib.GlobusRegex rgx = new GlobusRegex(); string urldata = System.Text.RegularExpressions.Regex.Split(System.Text.RegularExpressions.Regex.Split(getPinPageSource, "sourceFlagWrapper")[1], "</a>")[0]; linkurl = rgx.GetHrefUrlTag(urldata).Replace("href=\"", string.Empty); } catch (Exception ex) { } } else if (string.IsNullOrEmpty(linkurl)) { try { string urldata = System.Text.RegularExpressions.Regex.Split(System.Text.RegularExpressions.Regex.Split(getPinPageSource, "sourceFlagWrapper")[1], "</a>")[0]; string Datavalue = urldata.Substring(urldata.IndexOf("href=\\\"")); int startindex = Datavalue.IndexOf("href=\\\""); string start = Datavalue.Substring(startindex).Replace("href=\\\"", ""); int endindex = start.IndexOf("\\\""); string end = start.Substring(0, endindex); linkurl = end;// Datavalue.Substring(0, Datavalue.IndexOf("\\\">")).Replace("\\", string.Empty).Replace("\n", string.Empty).Replace(">", string.Empty).Replace("href=\"", string.Empty); } catch { }; } try { #region comment //Message = Message.Replace(" ", "+"); //string PostData = "source_url=%2Fpopular%2F%3Fpage%3D" + NumberOfPage + "&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + (Message.Replace(" ", "+")) + "%22%2C%22link%22%3A%22" + (Uri.EscapeDataString(linkurl)) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + pinterestAccountManager.App_version + "%22%7D%7D&module_path=App()%3EFeedPage()%3EGrid(resource%3DCategoryFeedResource(feed%3Dpopular))%3EGridItems(resource%3DCategoryFeedResource(feed%3Dpopular))%3EPin(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(submodule%3D%5Bobject+Object%5D%2C+color%3Dprimary%2C+text%3DPin+it%2C+primary_on_hover%3Dtrue%2C+extra_options%3D%5Bobject+Object%5D%2C+tagName%3Dbutton%2C+has_icon%3Dtrue%2C+show_text%3Dfalse%2C+ga_category%3Drepin_create%2C+class_name%3DrepinSmall%2C+require_auth%3Dtrue)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + "%2C+view_type%3Drepin)))"; //string PostData = "source_url=%2Fpopular%2F%3Fpage%3D" + "1" + "&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + (Message.Replace(" ", "+").Replace("\t", "+")) + "%22%2C%22link%22%3A%22" + (Uri.EscapeDataString(linkurl)) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + pinterestAccountManager.App_version + "%22%7D%7D&module_path=App()%3EFeedPage()%3EGrid(resource%3DCategoryFeedResource(feed%3Dpopular))%3EGridItems(resource%3DCategoryFeedResource(feed%3Dpopular))%3EPin(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(submodule%3D%5Bobject+Object%5D%2C+color%3Dprimary%2C+text%3DPin+it%2C+primary_on_hover%3Dtrue%2C+extra_options%3D%5Bobject+Object%5D%2C+tagName%3Dbutton%2C+has_icon%3Dtrue%2C+show_text%3Dfalse%2C+ga_category%3Drepin_create%2C+class_name%3DrepinSmall%2C+require_auth%3Dtrue)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + "%2C+view_type%3Drepin)))"; //string PostData = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + AddDesc + "%22%2C%22link%22%3A%22"+ linkurl +"%2F2011%2F11%2F01%2Fcaribbean-sangria%2F%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22520306563170537867%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%227d82d10%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D520306563170537867))%3EPinActionBar(resource%3DPinResource(id%3D520306563170537867))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D520306563170537867)))"; //string postdata = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + Message.Replace(" ","+") + "%22%2C%22link%22%3A%22" + Uri.EscapeDataString(linkurl) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22"+ PinId +"%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + pinterestAccountManager.App_version + "%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D" + PinId + "))%3EPinActionBar(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + ")))"; // string postdata = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + (Message.Replace(" ", "+")) + "%22%2C%22link%22%3A%22" + Uri.EscapeDataString(linkurl) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + pinterestAccountManager.App_version + "%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D" + PinId + "))%3EPinActionBar(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + ")))"; //My Commented //string postdata1 = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22Message%22%2C%22link%22%3A%22" + Uri.EscapeDataString(linkurl) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + pinterestAccountManager.App_version + "%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D" + PinId + "))%3EPinActionBar(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + ")))"; // Have to add Message below here //string postdata1 = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + myMessage + "%22%2C%22link%22%3A%22" + Uri.EscapeDataString(linkurl) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + pinterestAccountManager.App_version + "%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D" + PinId + "))%3EPinActionBar(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + ")))"; //source_url=%2Fpin%2F347410558725937380%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%2297460848125134285%22%2C%22description%22%3A%22Butterfly+tattoo%22%2C%22link%22%3A%22http%3A%2F%2Fwww.ruthtattooideas.com%2F2013%2F07%2F23%2Fbutterfly-tattoo%2F%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22347410558725937380%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%228c9869b%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D347410558725937380))%3EPinActionBar(resource%3DPinResource(id%3D347410558725937380))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D347410558725937380))) // string postdataa = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22description%22%3A%22" + (Message.Replace(" ", "+")) + "%22%2C%22link%22%3A%22" + SourceUrl + "%22%2C%22id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22" + pinterestAccountManager.App_version + "%3A%226bbb94a%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D" + PinId + "))%3ECloseupContent(resource%3DPinResource(id%3D381539399653733768))%3EPin(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(module%3DPinEdit)%23Modal(module%3DPinEdit(resource%3DPinResource(id%3D" + PinId + ")))"; //string afterposting = pinterestAccountManager.httpHelper.postFormDataProxy(new Uri("https://www.pinterest.com/resource/RepinResource/create/"), postdata1, "http://www.pinterest.com/pin/" + PinId + "/", "", 0, "", ""); #endregion string postdata1 = "source_url=%2Fpin%2F" + PinId + "%2F&data=%7B%22options%22%3A%7B%22board_id%22%3A%22" + Board + "%22%2C%22description%22%3A%22" + myMessage + "%22%2C%22link%22%3A%22" + Uri.EscapeDataString(linkurl) + "%22%2C%22is_video%22%3Afalse%2C%22pin_id%22%3A%22" + PinId + "%22%7D%2C%22context%22%3A%7B%22app_version%22%3A%22" + objPinUser.App_version + "%22%2C%22https_exp%22%3Afalse%7D%7D&module_path=App()%3ECloseup(resource%3DPinResource(id%3D" + PinId + "))%3EPinActionBar(resource%3DPinResource(id%3D" + PinId + "))%3EShowModalButton(module%3DPinCreate)%23Modal(module%3DPinCreate(resource%3DPinResource(id%3D" + PinId + ")))"; string afterposting = objPinUser.globusHttpHelper.postFormDataProxy(new Uri("https://www.pinterest.com/resource/RepinResource/create/"), postdata1, "http://www.pinterest.com/pin/" + PinId + "/", "", 0, "", ""); if (!afterposting.Contains("<div>Uh oh! Something went wrong.")) { return true; } return false; } catch (Exception ex) { }; } catch (Exception ex) { } } if (!string.IsNullOrEmpty(RepinPagesource)) { try { if (!string.IsNullOrEmpty(Globals.ItemSelect)) { GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " In " + Globals.ItemSelect + " is Done. ]"); } else { GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " is Done. ]"); } //string RepinDetails =pinterestAccountManager.Username + "," + PinId.Replace(",","") + "," + boardId.Replace(",","") + "," + myMessage.Replace(",",""); //GlobusFileHelper.AppendStringToTextfileNewLine(RepinDetails, ApplicationData.path_Repin); } catch { }; return true; } else { GlobusLogHelper.log.Info(" => [ Repining " + PinId + " For " + objPinUser.Username + " is Failed. ]"); return false; } // Log("[ " + DateTime.Now + " ] => [ Repining " + PinId + " For " + pinterestAccountManager.Username + " ]"); //string getPinPageSource = pinterestAccountManager.httpHelper.getHtmlfromUrl(new Uri("http://www.pinterest.com/" + PinId + "/"), "", "", "");//(new Uri("http://www.pinterest.com/pin/" + PinId + "/"), "", "", ""); //string getPinPageSource = pinterestAccountManager.httpHelper.getHtmlfromUrl(new Uri(PinId), "", "", ""); } } catch { }; return false; } catch (Exception Ex) { return false; } }