private void ScrapeKeywordSeacrh() { try { TwitterDataScrapper TweetData = new TwitterDataScrapper(); txtRecords.Invoke(new MethodInvoker(delegate { if (!string.IsNullOrEmpty(txtRecords.Text.Trim()) && NumberHelper.ValidateNumber(txtRecords.Text.Trim())) { TwitterDataScrapper.noOfRecords = Convert.ToInt32(txtRecords.Text.Trim()); if (TwitterDataScrapper.noOfRecords == 0) { AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Do not put Zero value ]"); AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Default number of records is 100 ]"); TwitterDataScrapper.noOfRecords = 100; } } else { AddToScrapeLogs("[ " + DateTime.Now + " ] => [ please enter value in number of users ]"); return; } })); AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Scrape by Keyword ]"); //List<TwitterDataScrapper.StructTweetIDs> data = TweetData.GetTweetData(txtScrapeKeyword.Text); //TweetData.logEvents.addToLogger += new EventHandler(DataScraperlogger_addToLogger); TweetData.logEvents.addToLogger += new EventHandler(DataScraperlogger_addToLogger); List<TwitterDataScrapper.StructTweetIDs> data = new List<TwitterDataScrapper.StructTweetIDs>(); foreach (string itemKeyword in lstSearchByKeywords) { if (!chkSearchByKeyWordByPeople.Checked) { data = TweetData.NewKeywordStructDataForSearchByKeyword(itemKeyword.Trim()); data = data.Distinct().ToList(); } else { data = TweetData.NewKeywordStructDataSearchByPeople(itemKeyword.Trim()); } } //TweetData.logEvents.addToLogger -= new EventHandler(DataScraperlogger_addToLogger); TweetData.logEvents.addToLogger -= new EventHandler(DataScraperlogger_addToLogger); data = DistinctDataList(data); if (!(data.Count() > 0)) { //AddToScrapeLogs("Request Not Completed"); AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Requesting For 100 User ids ]"); data = TweetData.NewKeywordStructData(txtScrapeKeyword.Text); } // AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + data.Count + " User ids Scraped ]"); AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Please Wait Till Data Is Retrieving ]"); #region commentedRegion //if (!chkSearchByKeyWordByPeople.Checked) //{ // if (!File.Exists(Globals.Path_KeywordScrapedListData + "-" + txtScrapeKeyword.Text + ".csv")) // { // GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS", Globals.Path_KeywordScrapedListData + "-" + txtScrapeKeyword.Text + ".csv"); // } // foreach (TwitterDataScrapper.StructTweetIDs item in data) // { // string ProfileName = string.Empty; // string Location = string.Empty; // string Bio = string.Empty; // string website = string.Empty; // string NoOfTweets = string.Empty; // string Followers = string.Empty; // string Followings = string.Empty; // ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr(); // GlobusHttpHelper HttpHelper = new GlobusHttpHelper(); // string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + item.username__Tweet_User), "", ""); // string Responce = ProfilePageSource; // #region Convert HTML to XML // string xHtml = objChilkat.ConvertHtmlToXml(Responce); // Chilkat.Xml xml = new Chilkat.Xml(); // xml.LoadXml(xHtml); // Chilkat.Xml xNode = default(Chilkat.Xml); // Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); // #endregion // int counterdata = 0; // xBeginSearchAfter = null; // string dataDescription = string.Empty; // //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field"); // xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name"); // while ((xNode != null)) // { // xBeginSearchAfter = xNode; // if (counterdata == 0) // { // ProfileName = xNode.AccumulateTagContent("text", "script|style"); // counterdata++; // } // else if (counterdata == 1) // { // website = xNode.AccumulateTagContent("text", "script|style"); // if (website.Contains("Twitter Status")) // { // website = "N/A"; // } // counterdata++; // } // else // { // break; // } // //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field"); // xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor"); // } // xBeginSearchAfter = null; // dataDescription = string.Empty; // xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field"); // while ((xNode != null)) // { // xBeginSearchAfter = xNode; // Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim(); // break; // } // xBeginSearchAfter = null; // dataDescription = string.Empty; // //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "location profile-field"); // xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field"); // while ((xNode != null)) // { // xBeginSearchAfter = xNode; // Location = xNode.AccumulateTagContent("text", "script|style"); // break; // } // int counterData = 0; // xBeginSearchAfter = null; // dataDescription = string.Empty; // //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "data-element-term", "tweet_stats"); // xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav"); // while ((xNode != null)) // { // xBeginSearchAfter = xNode; // if (counterData == 0) // { // NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty); // counterData++; // } // else if (counterData == 1) // { // Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty); // counterData++; // } // else if (counterData == 2) // { // Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty); // counterData++; // } // else // { // break; // } // //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav"); // xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor"); // } // if (!string.IsNullOrEmpty(item.username__Tweet_User) && item.ID_Tweet_User != "null") // { // string Id_user = item.ID_Tweet_User.Replace("}]", string.Empty).Trim(); // Globals.lstScrapedUserIDs.Add(Id_user); // GlobusFileHelper.AppendStringToTextfileNewLine(Id_user + "," + item.username__Tweet_User + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + "-" + txtScrapeKeyword.Text + ".csv"); // AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + Id_user + "," + item.username__Tweet_User + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]"); // } // } //} #endregion //AddToScrapeLogs("Retrieving data"); AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Adding Data To DataBase ]"); Globals.lstScrapedUserIDs = Globals.lstScrapedUserIDs.Distinct().ToList(); thread_AddingKeywordScrape = new Thread(() => { foreach (TwitterDataScrapper.StructTweetIDs item in data) { if (!string.IsNullOrEmpty(item.username__Tweet_User) && item.ID_Tweet_User != "null") { //AddToScrapeLogs(item.ID_Tweet_User); clsDBQueryManager DataBase = new clsDBQueryManager(); DataBase.InsertOrUpdateScrapeSetting(item.ID_Tweet_User, item.username__Tweet_User, item.ID_Tweet); } } AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Exported location :- " + Globals.Path_KeywordScrapedList + " ]"); AddToScrapeLogs("[ " + DateTime.Now + " ] => [ PROCESS COMPLETED ]"); AddToScrapeLogs("------------------------------------------------------------------------------------------------------------------------------------------"); }); thread_AddingKeywordScrape.Start(); if (Globals.IsDirectedFromFollower) { Thread.Sleep(1000); Globals.IsDirectedFromFollower = false; AddToLog_Follower("[ " + DateTime.Now + " ] => [ " + data.Count + " User ids Scraped and Added To Follow List ]"); Tb_AccountManager.Invoke(new MethodInvoker(delegate { Tb_AccountManager.SelectedIndex = 2; })); //tabMain.SelectedIndex = 2; } } catch (Exception ex) { Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeKeywordSeacrh() --> " + ex.Message, Globals.Path_ScrapeUsersErroLog); Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeKeywordSeacrh() --> " + ex.Message, Globals.Path_TwtErrorLogs); } }