private void ScrapeKeywordSeacrh()
{
try
{
TwitterDataScrapper TweetData = new TwitterDataScrapper();
if (!string.IsNullOrEmpty(txtRecords.Text) && NumberHelper.ValidateNumber(txtRecords.Text))
{
TweetData.noOfRecords = Convert.ToInt32(txtRecords.Text);
}
else
{
TweetData.noOfRecords = 100;
}
//List<TwitterDataScrapper.StructTweetIDs> data = TweetData.GetTweetData(txtScrapeKeyword.Text);
List<TwitterDataScrapper.StructTweetIDs> data = TweetData.KeywordStructData(txtScrapeKeyword.Text);
data = DistinctDataList(data);
if (!(data.Count() > 0))
{
AddToScrapeLogs("Request Not Complted");
AddToScrapeLogs("Requesting For 100 USer ids");
data = TweetData.GetTweetData(txtScrapeKeyword.Text);
}
AddToScrapeLogs(data.Count + " User ids Scraped ");
AddToScrapeLogs("Please Wait Till Data Is Retrieving");
int counter = 0;
if (!File.Exists(Globals.Path_KeywordScrapedList))
{
GlobusFileHelper.AppendStringToTextfileNewLine("Keyword , User-id , Username" , Globals.Path_KeywordScrapedList);
}
foreach (TwitterDataScrapper.StructTweetIDs item in data)
{
if (!string.IsNullOrEmpty(item.username__Tweet_User) && item.ID_Tweet_User != "null")
{
Globals.lstScrapedUserIDs.Add(item.ID_Tweet_User);
GlobusFileHelper.AppendStringToTextfileNewLine(txtScrapeKeyword.Text + "," + item.ID_Tweet_User + "," + item.username__Tweet_User, Globals.Path_KeywordScrapedList);
//AddToScrapeLogs(item.ID_Tweet_User);
}
}
//AddToScrapeLogs("Retrieving data");
AddToScrapeLogs("Adding Data To DataBase");
Globals.lstScrapedUserIDs = Globals.lstScrapedUserIDs.Distinct().ToList();
if (!File.Exists(Globals.Path_KeywordScrapedList))
{
GlobusFileHelper.AppendStringToTextfileNewLine("KEYWORD:USER ID:USERNAME ", Globals.Path_KeywordScrapedList);
}
new Thread(() =>
{
foreach (TwitterDataScrapper.StructTweetIDs item in data)
{
if (!string.IsNullOrEmpty(item.username__Tweet_User) && item.ID_Tweet_User != "null")
{
AddToScrapeLogs(item.ID_Tweet_User);
clsDBQueryManager DataBase = new clsDBQueryManager();
DataBase.InsertOrUpdateScrapeSetting(item.ID_Tweet_User, item.username__Tweet_User, item.ID_Tweet);
}
}
}).Start();
if (Globals.IsDirectedFromFollower)
{
Thread.Sleep(1000);
Globals.IsDirectedFromFollower = false;
AddToLog_Follower(data.Count + " User ids Scraped and Added To Follow List");
tabMain.Invoke(new MethodInvoker(delegate
{
tabMain.SelectedIndex = 2;
}));
//tabMain.SelectedIndex = 2;
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeKeywordSeacrh() --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeKeywordSeacrh() --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
private void threadStartScrape()
{
List<string> lst_structTweetFollowersIDs = new List<string>();
List<string> lst_structTweetFollowingsIds = new List<string>();
GlobusHttpHelper globusHttpHelper = new GlobusHttpHelper();
string user_id = string.Empty;
foreach (string keyword in lstscrapeUsername)
{
if (!GlobusRegex.ValidateNumber(keyword))//(!IsItNumber(user_id_toFollow))
{
user_id = TwitterDataScrapper.GetUserIDFromUsername(keyword);
}
else
{
user_id = keyword;
}
TwitterDataScrapper dataScrapeer = new TwitterDataScrapper();
if (chkboxScrapeFollowers.Checked)
{
try
{
if (!File.Exists(Globals.Path_ScrapedFollowersList))
{
GlobusFileHelper.AppendStringToTextfileNewLine("User_ID , FollowersUserID", Globals.Path_ScrapedFollowersList);
}
lst_structTweetFollowersIDs = dataScrapeer.GetFollowers(user_id);
//lst_structTweetFollowingsIds = dataScrapeer.GetFollowings(user_id);
AddToScrapeLogs("Added " + lst_structTweetFollowersIDs.Count + " Followers to list");
//AddToScrapeLogs("Added " + lst_structTweetFollowingsIds.Count + " Followings to list");
foreach (string data in lst_structTweetFollowersIDs)
{
try
{
Globals.lstScrapedUserIDs.Add(data);
GlobusFileHelper.AppendStringToTextfileNewLine(user_id + "," + data, Globals.Path_ScrapedFollowersList);
AddToScrapeLogs(data);
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowersIDs foreach --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowersIDs foreach --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
//AddToScrapeLogs("Added " + lst_structTweetFollowersIDs.Count + " Followers from User: "******"Data Exported to " + Globals.Path_ScrapedFollowersList);
if (Globals.IsDirectedFromFollower)
{
AddToLog_Follower("Added " + lst_structTweetFollowersIDs.Count + " Followers from User: "******" --> Error --> btnScrapeUser_Click() -- chkboxScrapeFollowers.Checked --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> btnScrapeUser_Click() -- chkboxScrapeFollowers.Checked --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
if (chkboxScrapeFollowings.Checked)
{
try
{
if (!File.Exists(Globals.Path_ScrapedFollowingsList))
{
GlobusFileHelper.AppendStringToTextfileNewLine("User_ID , FollowingsUserID", Globals.Path_ScrapedFollowingsList);
}
lst_structTweetFollowingsIds = dataScrapeer.GetFollowings(user_id);
AddToScrapeLogs("Added " + lst_structTweetFollowingsIds.Count + " Followings to list");
foreach (string data in lst_structTweetFollowingsIds)
{
try
{
Globals.lstScrapedUserIDs.Add(data);
GlobusFileHelper.AppendStringToTextfileNewLine(user_id + "," + data, Globals.Path_ScrapedFollowingsList);
AddToScrapeLogs(data);
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowingsIds foreach --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowingsIds foreach --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
AddToScrapeLogs("Added " + lst_structTweetFollowingsIds.Count + " Followings from User: "******"Data Exported to " + Globals.Path_ScrapedFollowingsList);
if (Globals.IsDirectedFromFollower)
{
AddToLog_Follower("Added " + lst_structTweetFollowingsIds.Count + " Followings from User: "******" --> Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowingsIds foreach --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowingsIds foreach --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
Globals.lstScrapedUserIDs = Globals.lstScrapedUserIDs.Distinct().ToList();
////new Thread(() =>
////{
//foreach (string data in lst_structTweetFollowersIDs)
//{
// try
// {
// clsDBQueryManager DataBase = new clsDBQueryManager();
// DataBase.InsertOrUpdateScrapeSetting(data, "");
// }
// catch (Exception ex)
// {
// }
//}
//// }
////).Start();
//AddToScrapeLogs("Added " + lst_structTweetFollowingsIds.Count + " Followings from User: "******"" , "");
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> lstScrapedUserIDs --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> lstScrapedUserIDs --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
}
).Start();
}
private void threadStartScrape()
{
TwitterDataScrapper dataScrapeer = new TwitterDataScrapper();
try
{
List<string> lst_structTweetFollowersIDs = new List<string>();
List<string> lst_structTweetFollowingsIds = new List<string>();
GlobusHttpHelper globusHttpHelper = new GlobusHttpHelper();
string user_id = string.Empty;
int counter = 0;
TweetAccountManager TweetLogin = new TweetAccountManager();
string ReturnStatus = string.Empty;
Globals.IsMobileVersion = true;
if (!Globals.IsMobileVersion)
{
foreach (KeyValuePair<string, TweetAccountManager> item in TweetAccountContainer.dictionary_TweetAccount)
{
if (counter > TweetAccountContainer.dictionary_TweetAccount.Count)
{
counter = 0;
}
TweetLogin = new TweetAccountManager();
TweetLogin.Username = item.Key;
TweetLogin.Password = item.Value.Password;
TweetLogin.proxyAddress = item.Value.proxyAddress;
TweetLogin.proxyPort = item.Value.proxyPort;
TweetLogin.proxyUsername = item.Value.proxyUsername;
TweetLogin.proxyPassword = item.Value.proxyPassword;
TweetLogin.Login();
if (!TweetLogin.IsLoggedIn)
{
continue;
}
else
{
globusHttpHelper = TweetLogin.globusHttpHelper;
counter++;
break;
}
}
if (!TweetLogin.IsLoggedIn)
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Please Upload Atleast One Working Account To Get Details ]");
return;
}
}
foreach (string keyword in lstscrapeUsername)
{
dataScrapeer.CounterDataNo = 500;
txtlimitScrapeUsers.Invoke(new MethodInvoker(delegate
{
if (!string.IsNullOrEmpty(txtlimitScrapeUsers.Text) && NumberHelper.ValidateNumber(txtlimitScrapeUsers.Text))
{
dataScrapeer.CounterDataNo = Convert.ToInt32(txtlimitScrapeUsers.Text);
}
}));
if (counter > TweetAccountContainer.dictionary_TweetAccount.Count)
{
counter = 0;
}
if (chkboxScrapeFollowers.Checked)
{
try
{
if (!File.Exists(Globals.Path_ScrapedFollowersList))
{
// GlobusFileHelper.AppendStringToTextfileNewLine("Name/Id , FollowersUserID , Followers User Name", Globals.Path_ScrapedFollowersList);
GlobusFileHelper.AppendStringToTextfileNewLine("Name/Id , Followers User Name", Globals.Path_ScrapedFollowersList);
}
try
{
dataScrapeer.logEvents.addToLogger += new EventHandler(DataScraperlogger_addToLogger);
if (!Globals.IsMobileVersion)
{
lst_structTweetFollowersIDs = dataScrapeer.GetFollowers_New(keyword.Trim(), out ReturnStatus, ref globusHttpHelper);
}
else
{
lst_structTweetFollowersIDs = dataScrapeer.GetFollowers_New_ForMobileVersion(keyword.Trim(), out ReturnStatus, ref globusHttpHelper);
}
dataScrapeer.logEvents.addToLogger -= new EventHandler(DataScraperlogger_addToLogger);
}
catch (Exception ex)
{
}
if (lst_structTweetFollowersIDs.Count > 0)
{
//AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Adding To DataBase and File ]");
//AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Added " + lst_structTweetFollowersIDs.Count + " Followers to list ]");
//foreach (string data in lst_structTweetFollowersIDs)
//{
// try
// {
// string[] arr_data = data.Split(':');
// Globals.lstScrapedUserIDs.Add(arr_data[0]);
// GlobusFileHelper.AppendStringToTextfileNewLine(keyword + "," + arr_data[0] + "," + arr_data[1], Globals.Path_ScrapedFollowersList);
// }
// catch (Exception ex)
// {
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowersIDs foreach --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowersIDs foreach --> " + ex.Message, Globals.Path_TwtErrorLogs);
// }
//}
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Added " + lst_structTweetFollowersIDs.Count + " Followers from User: "******" ]");
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Data Exported to " + Globals.Path_ScrapedFollowersList);
if (Globals.IsDirectedFromFollower)
{
AddToLog_Follower("[ " + DateTime.Now + " ] => [ Added " + lst_structTweetFollowersIDs.Count + " Followers from User: "******" ]");
Thread.Sleep(1000);
}
}
else if (ReturnStatus.Contains("Sorry, that page does not exist"))
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Sorry, That User does not exist =>" + ReturnStatus.Split(':')[1] + " ]");
continue;
}
else if (ReturnStatus == "Account is Suspended. ")
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Account " + keyword + " is Suspended. ]");
}
else if (ReturnStatus == "Error")
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Rate Limit Exceeded.Please Try After Some Time ]");
}
else
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + keyword + " User does not exist ]");
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> btnScrapeUser_Click() -- chkboxScrapeFollowers.Checked --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> btnScrapeUser_Click() -- chkboxScrapeFollowers.Checked --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
if (chkboxScrapeFollowings.Checked)
{
try
{
if (!File.Exists(Globals.Path_ScrapedFollowingsList))
{
// GlobusFileHelper.AppendStringToTextfileNewLine("Name/Id , FollowingsUserID , Following User Name", Globals.Path_ScrapedFollowingsList);
GlobusFileHelper.AppendStringToTextfileNewLine("Name/Id , Following User Name", Globals.Path_ScrapedFollowingsList);
}
string returnStaus = string.Empty;
string Screen_name = string.Empty;
dataScrapeer.logEvents.addToLogger += new EventHandler(DataScraperlogger_addToLogger);
// GlobusHttpHelper globusHttpHelper1 = new GlobusHttpHelper();
if (!Globals.IsMobileVersion)
{
lst_structTweetFollowingsIds = dataScrapeer.GetFollowings_New(keyword.Trim(), out returnStaus, ref globusHttpHelper);
}
else
{
lst_structTweetFollowingsIds = dataScrapeer.GetFollowings_NewForMobileVersion(keyword.Trim(), out returnStaus, ref globusHttpHelper);
}
if (lst_structTweetFollowingsIds.Count > 0)
{
if (lst_structTweetFollowingsIds.Count > 0)
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Added " + lst_structTweetFollowingsIds.Count + " Followings from User: "******" ]");
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Data Exported to " + Globals.Path_ScrapedFollowingsList + " ]");
if (Globals.IsDirectedFromFollower)
{
AddToLog_Follower("[ " + DateTime.Now + " ] => [ Added " + lst_structTweetFollowingsIds.Count + " Followings from User: "******" ]");
Thread.Sleep(1000);
//Tb_AccountManager.Invoke(new MethodInvoker(delegate
//{
// Tb_AccountManager.SelectedIndex = 2;
//}));
//tabMain.SelectedIndex = 2;
}
}
else if (returnStaus == "Error")
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Rate Limit Exceeded.Please Try After Some Time ]");
break;
}
}
else
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + keyword + " User does not have any followings ]");
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowingsIds foreach --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> btnScrapeKeyword_Click() -- lst_structTweetFollowingsIds foreach --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
Globals.lstScrapedUserIDs = Globals.lstScrapedUserIDs.Distinct().ToList();
}
Globals.IsMobileVersion = false;
new Thread(() =>
{
try
{
List<string> temp = new List<string>();
foreach (string item in Globals.lstScrapedUserIDs)
{
temp.Add(item);
}
foreach (string data in temp)
{
try
{
clsDBQueryManager DataBase = new clsDBQueryManager();
DataBase.InsertOrUpdateScrapeSetting(data, "", "");
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> lstScrapedUserIDs --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> lstScrapedUserIDs --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}
}
catch (Exception)
{
}
}).Start();
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ PROCESS COMPLETED ]");
AddToScrapeLogs("------------------------------------------------------------------------------------------------------------------------------------------");
//if (IsUserScrapedDatalist)
//{
// Tb_AccountManager.Invoke(new MethodInvoker(delegate
// {
// Tb_AccountManager.SelectedIndex = 0;
// //Tb_AccountManager.SelectedTab.Name = "tabFollower";
// }));
//}
}
catch (Exception)
{
}
finally
{
IsStart_ScrapUser = true;
dataScrapeer.logEvents.addToLogger -= new EventHandler(DataScraperlogger_addToLogger);
}
}
private void ScrapeKeywordSeacrh()
{
TwitterDataScrapper TweetData = new TwitterDataScrapper();
TweetData.noOfRecords = 1000;
List<TwitterDataScrapper.StructTweetIDs> data = TweetData.GetTweetData(txtScrapeKeyword.Text);
AddToScrapeLogs(data.Count + " User ids Scraped ");
foreach (TwitterDataScrapper.StructTweetIDs item in data)
{
Globals.lstScrapedUserIDs.Add(item.ID_Tweet_User);
AddToScrapeLogs(item.ID_Tweet_User);
}
Globals.lstScrapedUserIDs = Globals.lstScrapedUserIDs.Distinct().ToList();
new Thread(() =>
{
foreach (TwitterDataScrapper.StructTweetIDs item in data)
{
clsDBQueryManager DataBase = new clsDBQueryManager();
DataBase.InsertOrUpdateScrapeSetting(item.ID_Tweet_User, item.username__Tweet_User);
}
}).Start();
if (Globals.IsDirectedFromFollower)
{
Thread.Sleep(1000);
Globals.IsDirectedFromFollower = false;
AddToLog_Follower(data.Count + " User ids Scraped and Added To Follow List");
tabMain.Invoke(new MethodInvoker(delegate
{
tabMain.SelectedIndex = 2;
}));
//tabMain.SelectedIndex = 2;
}
}
private void ScrapeKeywordSeacrh()
{
try
{
TwitterDataScrapper TweetData = new TwitterDataScrapper();
txtRecords.Invoke(new MethodInvoker(delegate
{
if (!string.IsNullOrEmpty(txtRecords.Text.Trim()) && NumberHelper.ValidateNumber(txtRecords.Text.Trim()))
{
TwitterDataScrapper.noOfRecords = Convert.ToInt32(txtRecords.Text.Trim());
if (TwitterDataScrapper.noOfRecords == 0)
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Do not put Zero value ]");
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Default number of records is 100 ]");
TwitterDataScrapper.noOfRecords = 100;
}
}
else
{
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ please enter value in number of users ]");
return;
}
}));
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Scrape by Keyword ]");
//List<TwitterDataScrapper.StructTweetIDs> data = TweetData.GetTweetData(txtScrapeKeyword.Text);
//TweetData.logEvents.addToLogger += new EventHandler(DataScraperlogger_addToLogger);
TweetData.logEvents.addToLogger += new EventHandler(DataScraperlogger_addToLogger);
List<TwitterDataScrapper.StructTweetIDs> data = new List<TwitterDataScrapper.StructTweetIDs>();
foreach (string itemKeyword in lstSearchByKeywords)
{
if (!chkSearchByKeyWordByPeople.Checked)
{
data = TweetData.NewKeywordStructDataForSearchByKeyword(itemKeyword.Trim());
data = data.Distinct().ToList();
}
else
{
data = TweetData.NewKeywordStructDataSearchByPeople(itemKeyword.Trim());
}
}
//TweetData.logEvents.addToLogger -= new EventHandler(DataScraperlogger_addToLogger);
TweetData.logEvents.addToLogger -= new EventHandler(DataScraperlogger_addToLogger);
data = DistinctDataList(data);
if (!(data.Count() > 0))
{
//AddToScrapeLogs("Request Not Completed");
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Requesting For 100 User ids ]");
data = TweetData.NewKeywordStructData(txtScrapeKeyword.Text);
}
// AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + data.Count + " User ids Scraped ]");
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Please Wait Till Data Is Retrieving ]");
#region commentedRegion
//if (!chkSearchByKeyWordByPeople.Checked)
//{
// if (!File.Exists(Globals.Path_KeywordScrapedListData + "-" + txtScrapeKeyword.Text + ".csv"))
// {
// GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS", Globals.Path_KeywordScrapedListData + "-" + txtScrapeKeyword.Text + ".csv");
// }
// foreach (TwitterDataScrapper.StructTweetIDs item in data)
// {
// string ProfileName = string.Empty;
// string Location = string.Empty;
// string Bio = string.Empty;
// string website = string.Empty;
// string NoOfTweets = string.Empty;
// string Followers = string.Empty;
// string Followings = string.Empty;
// ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr();
// GlobusHttpHelper HttpHelper = new GlobusHttpHelper();
// string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + item.username__Tweet_User), "", "");
// string Responce = ProfilePageSource;
// #region Convert HTML to XML
// string xHtml = objChilkat.ConvertHtmlToXml(Responce);
// Chilkat.Xml xml = new Chilkat.Xml();
// xml.LoadXml(xHtml);
// Chilkat.Xml xNode = default(Chilkat.Xml);
// Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
// #endregion
// int counterdata = 0;
// xBeginSearchAfter = null;
// string dataDescription = string.Empty;
// //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field");
// xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name");
// while ((xNode != null))
// {
// xBeginSearchAfter = xNode;
// if (counterdata == 0)
// {
// ProfileName = xNode.AccumulateTagContent("text", "script|style");
// counterdata++;
// }
// else if (counterdata == 1)
// {
// website = xNode.AccumulateTagContent("text", "script|style");
// if (website.Contains("Twitter Status"))
// {
// website = "N/A";
// }
// counterdata++;
// }
// else
// {
// break;
// }
// //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field");
// xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor");
// }
// xBeginSearchAfter = null;
// dataDescription = string.Empty;
// xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field");
// while ((xNode != null))
// {
// xBeginSearchAfter = xNode;
// Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim();
// break;
// }
// xBeginSearchAfter = null;
// dataDescription = string.Empty;
// //xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "location profile-field");
// xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field");
// while ((xNode != null))
// {
// xBeginSearchAfter = xNode;
// Location = xNode.AccumulateTagContent("text", "script|style");
// break;
// }
// int counterData = 0;
// xBeginSearchAfter = null;
// dataDescription = string.Empty;
// //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "data-element-term", "tweet_stats");
// xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");
// while ((xNode != null))
// {
// xBeginSearchAfter = xNode;
// if (counterData == 0)
// {
// NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty);
// counterData++;
// }
// else if (counterData == 1)
// {
// Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty);
// counterData++;
// }
// else if (counterData == 2)
// {
// Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty);
// counterData++;
// }
// else
// {
// break;
// }
// //xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav");
// xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
// }
// if (!string.IsNullOrEmpty(item.username__Tweet_User) && item.ID_Tweet_User != "null")
// {
// string Id_user = item.ID_Tweet_User.Replace("}]", string.Empty).Trim();
// Globals.lstScrapedUserIDs.Add(Id_user);
// GlobusFileHelper.AppendStringToTextfileNewLine(Id_user + "," + item.username__Tweet_User + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + "-" + txtScrapeKeyword.Text + ".csv");
// AddToScrapeLogs("[ " + DateTime.Now + " ] => [ " + Id_user + "," + item.username__Tweet_User + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]");
// }
// }
//}
#endregion
//AddToScrapeLogs("Retrieving data");
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Adding Data To DataBase ]");
Globals.lstScrapedUserIDs = Globals.lstScrapedUserIDs.Distinct().ToList();
thread_AddingKeywordScrape = new Thread(() =>
{
foreach (TwitterDataScrapper.StructTweetIDs item in data)
{
if (!string.IsNullOrEmpty(item.username__Tweet_User) && item.ID_Tweet_User != "null")
{
//AddToScrapeLogs(item.ID_Tweet_User);
clsDBQueryManager DataBase = new clsDBQueryManager();
DataBase.InsertOrUpdateScrapeSetting(item.ID_Tweet_User, item.username__Tweet_User, item.ID_Tweet);
}
}
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ Exported location :- " + Globals.Path_KeywordScrapedList + " ]");
AddToScrapeLogs("[ " + DateTime.Now + " ] => [ PROCESS COMPLETED ]");
AddToScrapeLogs("------------------------------------------------------------------------------------------------------------------------------------------");
});
thread_AddingKeywordScrape.Start();
if (Globals.IsDirectedFromFollower)
{
Thread.Sleep(1000);
Globals.IsDirectedFromFollower = false;
AddToLog_Follower("[ " + DateTime.Now + " ] => [ " + data.Count + " User ids Scraped and Added To Follow List ]");
Tb_AccountManager.Invoke(new MethodInvoker(delegate
{
Tb_AccountManager.SelectedIndex = 2;
}));
//tabMain.SelectedIndex = 2;
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> ScrapeKeywordSeacrh() --> " + ex.Message, Globals.Path_ScrapeUsersErroLog);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> ScrapeKeywordSeacrh() --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
}