public static void CreateProfileXml()
{
DBConnect proc = new DBConnect();
DataTable rs3;
DataTable rs2;
DataTable rs;
int i = 0;
int j = 0;
int k = 0;
rs = proc.getAllProfil();
rs2 = proc.getAllPoste();
rs3 = proc.getAllTeam();
// Create the root node of a new XML document.
Chilkat.Xml xml = new Chilkat.Xml();
// This is going to be a collection of contacts.
xml.Tag = "profiles";
// Create a child node to hold a contact record.
// Our "xml" object will now reference the new node.
while (i < rs.Rows.Count)
{
xml = xml.NewChild("profil", "");
// Create a "company" record. NewChild2 does not return the child node that is created
xml.NewChild2("id", Convert.ToString(rs.Rows[i]["idProfil"]));
xml.NewChild2("nom", rs.Rows[i]["nom"].ToString() + ", " + rs.Rows[i]["prenom"].ToString());
xml.NewChild2("email", rs.Rows[i]["email"].ToString());
xml.NewChild2("phone", rs.Rows[i]["phoneNumber"].ToString());
xml.NewChild2("seniority", Convert.ToString(rs.Rows[i]["seniority"]));
// Move up the tree and create another contact record.
// GetParent2 updates our internal reference to the parent.
xml.GetParent2();
i++;
}
// We can always get back to the root by calling GetRoot (or GetRoot2)
xml.GetRoot2();
// Add the tag that specifies our character encoding.
xml.Encoding = "utf-8";
// Save the document.
xml.SaveXml("profiles.xml");
//GÉNÉRATION DE POSTES.XML
xml = new Chilkat.Xml();
xml.Tag = "postes";
while (j < rs2.Rows.Count)
{
xml = xml.NewChild("poste", "");
xml.NewChild2("id", Convert.ToString(rs2.Rows[j]["idPoste"]));
xml.NewChild2("nom", rs2.Rows[j]["nom"].ToString());
xml.NewChild2("description", rs2.Rows[j]["description"].ToString());
xml.GetParent2();
j++;
}
// We can always get back to the root by calling GetRoot (or GetRoot2)
xml.GetRoot2();
// Add the tag that specifies our character encoding.
xml.Encoding = "utf-8";
// Save the document.
xml.SaveXml("postes.xml");
//GÉNÉRATION DE POSTES.XML
xml = new Chilkat.Xml();
xml.Tag = "teams";
while (k < rs3.Rows.Count)
{
xml = xml.NewChild("team", "");
xml.NewChild2("id", Convert.ToString(rs3.Rows[k]["idTeam"]));
xml.NewChild2("nom", rs3.Rows[k]["nom"].ToString());
xml.NewChild2("description", rs3.Rows[k]["description"].ToString());
xml.GetParent2();
k++;
}
// We can always get back to the root by calling GetRoot (or GetRoot2)
xml.GetRoot2();
// Add the tag that specifies our character encoding.
xml.Encoding = "utf-8";
// Save the document.
xml.SaveXml("teams.xml");
}
/// <summary> Get User details for While/blacklist User
/// Get User details for While/blacklist User
/// </summary>
/// <param name="username">User Screan Name OR User ID</param>
/// <returns>Details of Users</returns>
///
#region Get User details for While/blacklist User
//public static Dictionary<string, string> GetUserDetails(string username, out string status)
//{
//// ChilkatHttpHelpr httpHelper = new ChilkatHttpHelpr();
// Dictionary<string, string> dataLst = new Dictionary<string, string>();
//// try
//// {
//// string PageSource = null;
//// if (!string.IsNullOrEmpty(username) && NumberHelper.ValidateNumber(username))
//// {
//// //PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?user_id=" + username + "&include_entities=true");
//// }
//// else
//// {
//// //PageSource = httpHelper.GetHtml("https://api.twitter.com/1/users/show.json?screen_name=" + username + "&include_entities=true");
//// string url = string.Format("https://twitter.com/" + username);
//// PageSource = httpHelper.GetHtml(url);
//// }
//// if (PageSource.Contains("error\":\"User has been suspended"))
//// {
//// status = "User has been suspended";
//// return dataLst;
//// }
//// else if (PageSource.Contains("message\":\"Sorry, that page does not exist"))
//// {
//// status = "Sorry, that page does not exist";
//// return dataLst;
//// }
//// else if (PageSource.Contains("Rate limit exceeded. Clients may not make more than 150 requests per hour."))
//// {
//// status = "Rate limit exceeded. Clients may not make more than 150 requests per hour.";
//// return dataLst;
//// }
//// if (!string.IsNullOrEmpty(PageSource))
//// {
//// string id = string.Empty;
//// string name = string.Empty;
//// string screen_name = string.Empty;
//// string location = string.Empty;
//// string NoOfTweet = string.Empty;
//// string followers_count = string.Empty;
//// string friends_count = string.Empty;
//// string profile_image_url = string.Empty;
//// if (PageSource.Contains("data-user-id="))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "data-user-id=");
//// if (getDataInArr.Count() > 0)
//// {
//// foreach (var item in getDataInArr)
//// {
//// int startindex = item.IndexOf("<strong>");
//// int endindex = item.IndexOf("</strong>"); ;
//// NoOfTweet = item.Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
//// }
//// }
//// }
//// if (PageSource.Contains("tweet_stats"))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "tweet_stats");
//// if (getDataInArr.Count() > 0)
//// {
//// int startindex = getDataInArr[1].IndexOf("<strong>");
//// int endindex = getDataInArr[1].IndexOf("</strong>"); ;
//// NoOfTweet = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
//// }
//// }
//// if (PageSource.Contains("following_stats"))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "following_stats");
//// if (getDataInArr.Count() > 0)
//// {
//// int startindex = getDataInArr[1].IndexOf("<strong>");
//// int endindex = getDataInArr[1].IndexOf("</strong>"); ;
//// friends_count = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
//// }
//// }
//// if (PageSource.Contains("follower_stats"))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "follower_stats");
//// if (getDataInArr.Count() > 0)
//// {
//// int startindex = getDataInArr[1].IndexOf("<strong>");
//// int endindex = getDataInArr[1].IndexOf("</strong>");
//// followers_count = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
//// }
//// }
//// if (PageSource.Contains("data-url"))
//// {
//// String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(PageSource, "data-url");
//// if (getDataInArr.Count() > 0)
//// {
//// int startindex = getDataInArr[1].IndexOf("=\"");
//// int endindex = getDataInArr[1].IndexOf(">"); ;
//// profile_image_url = getDataInArr[1].Substring(startindex, endindex - startindex).Replace("target=\"_blank", string.Empty).Replace("\"", string.Empty).Replace("=", string.Empty).Trim();
//// }
//// }
//// //string id = jobj["id"].ToString().Replace("\"", string.Empty);
//// dataLst.Add("id", id);
//// //string name = jobj["name"].ToString().Replace("\"", string.Empty);
//// dataLst.Add("name", name);
//// //string screen_name = jobj["screen_name"].ToString().Replace("\"", string.Empty);
//// dataLst.Add("screen_name", screen_name);
//// //string location = jobj["location"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("location", location);
//// //string description = jobj["description"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("description", description);
//// //string followers_count = jobj["followers_count"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("followers_count", followers_count);
//// //string friends_count = jobj["friends_count"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("friends_count", friends_count);
//// //string statuses_count = jobj["statuses_count"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("statuses_count", statuses_count);
//// //string created_at = jobj["created_at"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("created_at", created_at);
//// //string time_zone = jobj["time_zone"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("time_zone", time_zone);
//// //string profile_image_url = jobj["profile_image_url"].ToString().Replace("\"", string.Empty);
//// //dataLst.Add("profile_image_url", profile_image_url);
//// }
//// }
//// catch (Exception ex)
//// {
//// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
//// Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + username + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
//// }
//// //Assign Status
// status = "Ok";
// return dataLst;
//}
public static Dictionary<string, string> GetUserDetails(string username, out string status)
{
TwitterDataScrapper twiterDataScrapper = new TwitterDataScrapper();
Dictionary<string, string> dataLst = new Dictionary<string, string>();
string id = string.Empty;
string name = string.Empty;
string screen_name = string.Empty;
string location = string.Empty;
string description = string.Empty;
string NoOfTweet = string.Empty;
string followers_count = string.Empty;
string following_count = string.Empty;
string profile_image_url = string.Empty;
//string ProfileName = string.Empty;
//string Location = string.Empty;
//string Bio = string.Empty;
string website = string.Empty;
//string NoOfTweets = string.Empty;
//string Followers = string.Empty;
//string Followings = string.Empty;
//try
//{
ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr();
GlobusHttpHelper HttpHelper = new GlobusHttpHelper();
string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + username), "", "");
string Responce = ProfilePageSource;
#region Convert HTML to XML
string xHtml = objChilkat.ConvertHtmlToXml(Responce);
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
//Get User Id From page
if (!(ProfilePageSource == null))
{
if (ProfilePageSource.Contains("data-user-id="))
{
try
{
String[] getDataInArr = System.Text.RegularExpressions.Regex.Split(ProfilePageSource, "data-user-id=");
if (getDataInArr.Count() > 0)
{
foreach (var item in getDataInArr)
{
if (item.Contains("<h1"))
{
int startindex = (1);
int endindex = item.IndexOf(">") - 1; ;
id = item.Substring(startindex, endindex - startindex).Replace("<strong>", string.Empty).Replace(",", string.Empty);
if (id.Contains("\n"))
{
string[] arr =System.Text.RegularExpressions.Regex.Split(id,"\"");
id=arr[0];
}
dataLst.Add("id", id);
break;
}
}
//whiteListedUserCount++;
}
}
catch (Exception)
{
}
}
}
//else
//{
// blackListedUserCount++;
//}
int counterdata = 0;
xBeginSearchAfter = null;
string dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNameTruncated-link u-textInheritColor js-nav js-action-profile-name");//"profile-field");
while ((xNode != null))
{
try
{
string str = xNode.GetXml();
xBeginSearchAfter = xNode;
if (counterdata == 1)
{
website = xNode.AccumulateTagContent("text", "script|style");
counterdata++;
}
else if (counterdata == 0)
{
name = xNode.AccumulateTagContent("text", "script|style");
dataLst.Add("name", name);
counterdata++;
}
else
{
break;
}
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNameTruncated-link u-textInheritColor js-nav js-action-profile-name");//"profile-field");
}
catch (Exception)
{
}
}
//Get Screen name and ID
xBeginSearchAfter = null;
//string dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "u-linkComplex-target");
while ((xNode != null))
{
string str = xNode.GetXml();
xBeginSearchAfter = xNode;
screen_name = xNode.AccumulateTagContent("text", "script|style").Replace("@", string.Empty);
dataLst.Add("screen_name", screen_name);
break;
}
xBeginSearchAfter = null;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//"location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
location = xNode.AccumulateTagContent("text", "script|style");
dataLst.Add("location", location);
break;
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
description = xNode.AccumulateTagContent("text", "script|style");
dataLst.Add("description", description);
break;
}
int counterData = 0;
xBeginSearchAfter = null;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");
xBeginSearchAfter = xNode;
if (counterData == 0)
{
NoOfTweet = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet",string.Empty);
dataLst.Add("statuses_count", NoOfTweet);
counterData++;
}
//dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
xBeginSearchAfter = xNode;
if (counterData == 1)
{
following_count = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty);
dataLst.Add("friends_count", following_count);
counterData++;
}
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
//while ((xNode != null))
//{
// //xBeginSearchAfter = xNode;
//if (counterData == 0)
//{
// NoOfTweet = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet",string.Empty);
// dataLst.Add("statuses_count", NoOfTweet);
// counterData++;
//}
xBeginSearchAfter = xNode;
if (counterData == 2)
{
followers_count = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty);
dataLst.Add("followers_count", followers_count);
counterData++;
}
//else
//{
// break;
//}
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//"js-nav");
// }
status = "Ok";
//}
//catch (Exception)
//{
// status = "";
//}
return dataLst;
}
public void CrawlingPageDataSource(string Url, ref GlobusHttpHelper HttpHelper)
{
if (SearchCriteria.starter)
{
if (SearchCriteria.starter)
{
try
{
Log("[ " + DateTime.Now + " ] => [ Start Parsing Process ]");
#region Data Initialization
string Industry = string.Empty;
string URLprofile = string.Empty;
string firstname = string.Empty;
string lastname = string.Empty;
string location = string.Empty;
string country = string.Empty;
string postal = string.Empty;
string phone = string.Empty;
string USERemail = string.Empty;
string code = string.Empty;
string education1 = string.Empty;
string education2 = string.Empty;
string titlecurrent = string.Empty;
string companycurrent = string.Empty;
string titlepast1 = string.Empty;
string companypast1 = string.Empty;
string titlepast2 = string.Empty;
string html = string.Empty;
string companypast2 = string.Empty;
string titlepast3 = string.Empty;
string companypast3 = string.Empty;
string titlepast4 = string.Empty;
string companypast4 = string.Empty;
string Recommendations = string.Empty;
string Connection = string.Empty;
string Designation = string.Empty;
string Website = string.Empty;
string Contactsettings = string.Empty;
string recomandation = string.Empty;
string titleCurrenttitle = string.Empty;
string titleCurrenttitle2 = string.Empty;
string titleCurrenttitle3 = string.Empty;
string titleCurrenttitle4 = string.Empty;
string Skill = string.Empty;
//string TypeOfProfile = "Public1";
string Finaldata = string.Empty;
#endregion
#region LDS_DataInitialization
string LDS_FirstName = string.Empty;
string LDS_LastName = string.Empty;
string LDS_UserProfileLink = string.Empty;
string LDS_HeadLineTitle = string.Empty;
string LDS_CurrentTitle = string.Empty;
string LDS_PastTitles = string.Empty;
string LDS_Loction = string.Empty;
string LDS_Country = string.Empty;
string LDS_Connection = string.Empty;
string LDS_Recommendations = string.Empty;
string LDS_SkillAndExpertise = string.Empty;
string LDS_Education = string.Empty;
string LDS_Experience = string.Empty;
string LDS_ProfileType = "Public";
string LDS_Groups = string.Empty;
string LDS_UserEmail = string.Empty;
string LDS_UserContactNumber = string.Empty;
string LDS_CurrentCompany = string.Empty;
string LDS_PastCompany = string.Empty;
string LDS_LoginID = string.Empty;
string LDS_Websites = string.Empty;
string LDS_Industry = string.Empty;
string companyCurrenttitle1 = string.Empty;
string companyCurrenttitle2 = string.Empty;
string companyCurrenttitle3 = string.Empty;
string companyCurrenttitle4 = string.Empty;
#endregion
#region Chilkat Initialization
Chilkat.Http http = new Chilkat.Http();
///Chilkat Http Request to be used in Http Post...
Chilkat.HttpRequest req = new Chilkat.HttpRequest();
Chilkat.HtmlUtil htmlUtil = new Chilkat.HtmlUtil();
// Any string unlocks the component for the 1st 30-days.
bool success = http.UnlockComponent("THEBACHttp_b3C9o9QvZQ06");
if (success != true)
{
System.Console.WriteLine(http.LastErrorText);
return;
}
http.CookieDir = "memory";
http.SendCookies = true;
http.SaveCookies = true;
html = HttpHelper.getHtmlfromUrl1(new Uri(Url));
html = htmlUtil.EntityDecode(html);
//// Convert the HTML to XML:
Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml();
Chilkat.HtmlToXml htmlToXml1 = new Chilkat.HtmlToXml();
Chilkat.HtmlToXml htmlToXml2 = new Chilkat.HtmlToXml();
success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O");
if ((success != true))
{
Console.WriteLine(htmlToXml.LastErrorText);
return;
}
string xHtml = null;
string xHtml1 = null;
//string xHtml2 = null;
htmlToXml.Html = html;
xHtml = htmlToXml.ToXml();
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
//// Iterate over all h1 tags:
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
#region for paRSING
List<string> list = new List<string>();
List<string> Grouplist = new List<string>();
List<string> listtitle = new List<string>();
List<string> Currentlist = new List<string>();
List<string> Skilllst = new List<string>();
list.Clear();
//new parshing code
List<string> TempFirstName = objChilkat.GetDataTagAttributewithId(html, "div", "name-container");
xBeginSearchAfter = null;
xNode = xml.SearchForTag(xBeginSearchAfter, "dt");
Grouplist.Clear();
xBeginSearchAfter = null;
#region parsergroup
xNode = xml.SearchForAttribute(xBeginSearchAfter, "div", "class", "group-data");
while ((xNode != null))
{
Finaldata = xNode.AccumulateTagContent("text", "/text");
Grouplist.Add(Finaldata);
string[] tempC1 = Regex.Split(Finaldata, " at ");
xBeginSearchAfter = xNode;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "div", "class", "group-data");
}
int groupcounter = 0;
string AllGRoup = string.Empty;
foreach (string item in Grouplist)
{
if (item.Contains("Join"))
{
if (groupcounter == 0)
{
LDS_Groups = item;
groupcounter++;
}
else
{
LDS_Groups = AllGRoup + ";" + item;
}
}
}
#endregion
#region parserSkill
xNode = xml.SearchForTag(xBeginSearchAfter, "dt");
Skilllst.Clear();
xBeginSearchAfter = null;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "div", "id", "profile-skills");
while ((xNode != null))
{
Finaldata = xNode.AccumulateTagContent("text", "/text");
if (Finaldata.Contains("extlib: _toggleclass"))
{
try
{
string[] Temp = Finaldata.Split(';');
LDS_SkillAndExpertise = Temp[4];
}
catch { }
}
else
{
try
{
LDS_SkillAndExpertise = Finaldata.Replace("Skills & Expertise", " ");
Skilllst.Add(Finaldata);
}
catch { }
}
xBeginSearchAfter = xNode;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "div", "id", "profile-skills");
}
if (LDS_SkillAndExpertise.Contains(" Endorsements LI.i18n.register('section_skills_person_endorsed_tmpl"))
{
LDS_SkillAndExpertise = string.Empty;
}
Skilllst.Distinct();
#endregion
#region UrlProfile
try
{
if (html.Contains("webProfileURL"))
{
int FirstPointForProfileURL = html.IndexOf("webProfileURL");
string FirstSubStringForProfileURL = html.Substring(FirstPointForProfileURL);
int SecondPointForProfileURL = FirstSubStringForProfileURL.IndexOf(">");
int ThirdPointForProfileURL = FirstSubStringForProfileURL.IndexOf("</a>");
string SecondSubStringForProfileURL = FirstSubStringForProfileURL.Substring(SecondPointForProfileURL, ThirdPointForProfileURL - SecondPointForProfileURL);
LDS_UserProfileLink = SecondSubStringForProfileURL.Replace(">", string.Empty);
//qm.AddProfileUrl(URLprofile, DateTime.Now.ToString(), "0");
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
try
{
string[] UrlFull = System.Text.RegularExpressions.Regex.Split(Url, "&authType");
LDS_UserProfileLink = UrlFull[0];
LDS_UserProfileLink = Url;
}
catch { }
#endregion
#region Connection
if (html.Contains("overview-connections"))
{
try
{
Connection = html.Substring(html.IndexOf("leo-module mod-util connections"), 500);
string[] Arr = Connection.Split('>');
string tempConnection = Arr[5].Replace("</strong", "").Replace(")</h3", "").Replace("(", "");
if (tempConnection.Length < 8)
{
LDS_Connection = tempConnection + "Connection";
}
else
{
LDS_Connection = string.Empty;
}
}
catch (Exception ex)
{
//overview-connections
try
{
LDS_Connection = html.Substring(html.IndexOf("overview-connections"), 50);
string[] Arr = Connection.Split('>');
string tempConnection = Arr[3].Replace("</strong", "").Replace(")</h3", "").Replace("(", "");
LDS_Connection = tempConnection + "Connection";
}
catch { }
}
}
#endregion
#region Recommendation
if (html.Contains("Recommendations"))
{
try
{
string[] rList = System.Text.RegularExpressions.Regex.Split(html, "Recommendations");
string[] R3List = rList[2].Split('\n');
string temprecomandation = R3List[4].Replace("</strong>", "").Replace("<strong>", "");
if (temprecomandation.Contains("recommended"))
{
LDS_Recommendations = temprecomandation;
}
else
{
LDS_Recommendations = "";
}
}
catch (Exception ex)
{
LDS_Recommendations = string.Empty;
}
}
#endregion
#region Websites
if (html.Contains("websites"))
{
try
{
string websitedem = html.Substring(html.IndexOf("websites"), 500);
string[] Arr = Regex.Split(websitedem, "href");
foreach (string item in Arr)
{
if (item.Contains("redir/redirect?url"))
{
string tempArr = item.Substring(item.IndexOf("name="), 50);
string[] temarr = tempArr.Split('\n');
LDS_Websites = temarr[1];
}
}
}
catch (Exception ex)
{
LDS_Websites = string.Empty;
}
}
#endregion
#region Getting Industry
try
{
string Industrytemp = html.Substring(html.IndexOf("Find users in this industry"), 100);
string[] TempIndustery = Industrytemp.Split('>');
LDS_Industry = TempIndustery[1].Replace("</strong", "").Replace("</a", "");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
#endregion
#region Getting First Name
try
{
if (html.Contains("given-name"))
{
int FirstPointForProfilename = html.IndexOf("given-name");
string FirstSubStringForProfilename = html.Substring(FirstPointForProfilename);
int SecondPointForProfilename = FirstSubStringForProfilename.IndexOf(">");
int ThirdPointForProfilename = FirstSubStringForProfilename.IndexOf("</span>");
string SecondSubStringForProfilename = FirstSubStringForProfilename.Substring(SecondPointForProfilename, ThirdPointForProfilename - SecondPointForProfilename);
LDS_FirstName = SecondSubStringForProfilename.Replace(">", string.Empty);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
#endregion
#region LastName
try
{
if (html.Contains("family-name"))
{
int FirstPointForProfilelastname = html.IndexOf("family-name");
string FirstSubStringForProfilelastname = html.Substring(FirstPointForProfilelastname);
int SecondPointForProfilelastname = FirstSubStringForProfilelastname.IndexOf(">");
int ThirdPointForProfilelastname = FirstSubStringForProfilelastname.IndexOf("</span>");
string SecondSubStringForProfilelastname = FirstSubStringForProfilelastname.Substring(SecondPointForProfilelastname, ThirdPointForProfilelastname - SecondPointForProfilelastname);
string templastname = SecondSubStringForProfilelastname.Replace(">", string.Empty);
if (templastname.Contains(","))
{
string[] arrylastname = templastname.Split(',');
LDS_LastName = arrylastname[0];
}
else
{
LDS_LastName = templastname;
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
#endregion
#region Designation Company Current
try
{
if (html.Contains("phonetic-full-name"))
{
int FirstPointForProfileCurrent = html.IndexOf("phonetic-full-name");
string FirstSubStringForProfileCurrent = html.Substring(FirstPointForProfileCurrent);
int SecondPointForProfileCurrent = FirstSubStringForProfileCurrent.IndexOf("display:block");
int ThirdPointForProfileCurrent = FirstSubStringForProfileCurrent.IndexOf("</p>");
string SecondSubStringForProfileCurrent = FirstSubStringForProfileCurrent.Substring(SecondPointForProfileCurrent, ThirdPointForProfileCurrent - SecondPointForProfileCurrent);
titlecurrent = SecondSubStringForProfileCurrent.Replace("\">", "").Replace("display:block", string.Empty).Replace("<strong class=\"highlight\"", string.Empty).Replace("</strong", string.Empty).Trim();
string[] tempCCurent = Regex.Split(titlecurrent, " at ");
LDS_HeadLineTitle = titlecurrent.Replace(",", ";");
LDS_CurrentCompany = tempCCurent[1].Replace(",", ";");
}
else if (html.Contains("<p class=\"title\""))
{
LDS_HeadLineTitle = html.Substring(html.IndexOf("<p class=\"title\""), 150);
string[] HeadLineTitle = LDS_HeadLineTitle.Split('>');
string tempHeadLineTitle = HeadLineTitle[1].Replace("\n", "").Replace(")</h3", "").Replace("</p", "");
LDS_HeadLineTitle = tempHeadLineTitle;
try
{
string[] tempCCurent = Regex.Split(tempHeadLineTitle, " at ");
LDS_HeadLineTitle = tempCCurent[0];
LDS_CurrentCompany = tempCCurent[1];
}
catch { }
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
#endregion
#region Education
try
{
if (html.Contains("summary-education"))
{
int FirstPointForProfileeducation1 = html.IndexOf("summary-education");
string FirstSubStringForProfileeducation1 = html.Substring(FirstPointForProfileeducation1);
int SecondPointForProfileeducation1 = FirstSubStringForProfileeducation1.IndexOf("<li>");
int ThirdPointForProfileeducation1 = FirstSubStringForProfileeducation1.IndexOf("</li>");
string SecondSubStringForProfileeducation1 = FirstSubStringForProfileeducation1.Substring(SecondPointForProfileeducation1, ThirdPointForProfileeducation1 - SecondPointForProfileeducation1);
education1 = SecondSubStringForProfileeducation1.Replace("<li>", string.Empty).Replace(",", string.Empty).Trim();
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
#endregion
#region Country
try
{
if (html.Contains("locality"))
{
int FirstPointForlocality = html.IndexOf("locality");
string FirstSubStringForlocality = html.Substring(FirstPointForlocality);
int SecondPointForlocality = FirstSubStringForlocality.IndexOf("location");
int ThirdPointForlocality = FirstSubStringForlocality.IndexOf("</a>");
string SecondSubStringForlocality = FirstSubStringForlocality.Substring(SecondPointForlocality, ThirdPointForlocality - SecondPointForlocality);
string temlocation = SecondSubStringForlocality.Replace("location", string.Empty).Replace(">", string.Empty).Replace('"', ' ');
string[] temp = temlocation.Split(',');
LDS_Loction = temp[0].Replace("<strong class= highlight", string.Empty).Replace("</strong", string.Empty);
LDS_Country = temp[1].Replace("<strong class= highlight", string.Empty).Replace("</strong", string.Empty);
// country = temp[1].Replace("</strong", string.Empty);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
#endregion
#region User Email
try
{
if (html.Contains("Email & Phone:"))
{
int FirstPointFortitlepast1 = html.IndexOf("abook-email");
string FirstSubStringFortitlepast1 = html.Substring(FirstPointFortitlepast1);
int SecondPointFortitlepast1 = FirstSubStringFortitlepast1.IndexOf("<a");
int ThirdPointFortitlepast1 = FirstSubStringFortitlepast1.IndexOf("</a>");
string SecondSubStringFortitlepast1 = FirstSubStringFortitlepast1.Substring(SecondPointFortitlepast1, ThirdPointFortitlepast1 - SecondPointFortitlepast1);
string[] tempEmail = SecondSubStringFortitlepast1.Split('>');
LDS_UserEmail = tempEmail[1];
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
#endregion
#region Type Of profile
try
{
if (html.Contains("profile-header"))
{
int FirstPointForProfileType = html.IndexOf("profile-header");
string FirstSubStringForProfileType = html.Substring(FirstPointForProfileType);
int SecondPointForProfileType = FirstSubStringForProfileType.IndexOf("class=\"n fn\"");
int ThirdPointForProfileType = FirstSubStringForProfileType.IndexOf("</span>");
string SecondSubStringForProfileType = FirstSubStringForProfileType.Substring(SecondPointForProfileType, ThirdPointForProfileType - SecondPointForProfileType);
string[] tempProfileType = SecondSubStringForProfileType.Split('>');
string ProfileType = tempProfileType[1];
LDS_ProfileType = ProfileType;
}
//<h1><span id="name" class="n fn">Private</span>
else if (html.Contains(" class=\"n fn\""))
{
try
{
string ProfileTypetemp = html.Substring(html.IndexOf("class=\"n fn\""), 20);
string[] TempProfileType = ProfileTypetemp.Split('>');
LDS_ProfileType = TempProfileType[1].Replace("</strong", "").Replace("</a", "");
}
catch { }
}
if (LDS_ProfileType != "Public")
{
LDS_ProfileType = "Private";
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
#endregion
#region PhonNumber
try
{
if (html.Contains("<dt>Phone:</dt>"))
{
int FirstPointFortitlepast1 = html.IndexOf("profile-personal");
string FirstSubStringFortitlepast1 = html.Substring(FirstPointFortitlepast1);
int SecondPointFortitlepast1 = FirstSubStringFortitlepast1.IndexOf("<p>");
int ThirdPointFortitlepast1 = FirstSubStringFortitlepast1.IndexOf("<span");
string SecondSubStringFortitlepast1 = FirstSubStringFortitlepast1.Substring(SecondPointFortitlepast1, ThirdPointFortitlepast1 - SecondPointFortitlepast1);
LDS_UserContactNumber = SecondSubStringFortitlepast1.Replace("<p>", string.Empty);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
#endregion
xNode = xml.SearchForTag(xBeginSearchAfter, "dt");
xBeginSearchAfter = xNode;
list.Clear();
#endregion
#region Regionfor PastCompney
try
{
if (html.Contains("summary-past"))
{
int FirstPointForPasttitle = html.IndexOf("summary-past");
string FirstSubStringForPasttitle = html.Substring(FirstPointForPasttitle);
int SecondPointForPasttitle = FirstSubStringForPasttitle.IndexOf("<li>");
int ThirdPointForPasttitle = FirstSubStringForPasttitle.IndexOf("summary-education");
string SecondSubStringForPasttitle = FirstSubStringForPasttitle.Substring(SecondPointForPasttitle, ThirdPointForPasttitle - SecondPointForPasttitle);
string FirstSubStringForPasttitlelast = htmlUtil.EntityDecode(SecondSubStringForPasttitle);
htmlToXml1.Html = FirstSubStringForPasttitlelast;
xHtml1 = htmlToXml1.ToXml();
Chilkat.Xml xml1 = new Chilkat.Xml();
xml1.LoadXml(xHtml1);
//// Iterate over all h1 tags:
Chilkat.Xml xNode1 = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter1 = default(Chilkat.Xml);
list.Clear();
string[] tempC1 = null;
xNode1 = xml1.SearchForTag(xBeginSearchAfter1, "li");
while ((xNode1 != null))
{
Finaldata = xNode1.AccumulateTagContent("text", "/text");
listtitle.Add(Finaldata);
// list.Add(Finaldata);
try
{
tempC1 = Regex.Split(Finaldata, " at ");
}
catch { }
if (tempC1 != null)
{
try
{
list.Add(tempC1[1]);
}
catch { }
}
xNode1 = xml1.SearchForTag(xBeginSearchAfter1, "li");
xBeginSearchAfter1 = xNode1;
}
if (listtitle.Count > 0 || list.Count > 0)
{
try
{
titlepast1 = listtitle[0] != null ? listtitle[0] : string.Empty;
titlepast2 = listtitle[1] != null ? listtitle[1] : string.Empty;
titlepast3 = listtitle[2] != null ? listtitle[2] : string.Empty;
titlepast4 = listtitle[3] != null ? listtitle[3] : string.Empty;
}
catch { }
try
{
companypast1 = list[0] != null ? list[0] : string.Empty;
companypast2 = list[1] != null ? list[1] : string.Empty;
companypast3 = list[2] != null ? list[2] : string.Empty;
companypast4 = list[3] != null ? list[3] : string.Empty;
}
catch { }
}
}
}
catch { };
list.Clear();
#endregion
#region Regionfor summary-current
try
{
if (html.Contains("summary-current"))
{
int FirstPointForCurrenttitle = html.IndexOf("summary-current");
string FirstSubStringForCurrenttitle = html.Substring(FirstPointForCurrenttitle);
int SecondPointForCurrenttitle = FirstSubStringForCurrenttitle.IndexOf("<li>");
int ThirdPointForCurrenttitle = FirstSubStringForCurrenttitle.IndexOf("summary-past");
string SecondSubStringForCurrenttitle = FirstSubStringForCurrenttitle.Substring(SecondPointForCurrenttitle, ThirdPointForCurrenttitle - SecondPointForCurrenttitle);
string FirstSubStringForCurrenttitlelast = htmlUtil.EntityDecode(SecondSubStringForCurrenttitle);
htmlToXml1.Html = FirstSubStringForCurrenttitlelast;
xHtml1 = htmlToXml1.ToXml();
Chilkat.Xml xml1 = new Chilkat.Xml();
xml1.LoadXml(xHtml1);
//// Iterate over all h1 tags:
Chilkat.Xml xNode1 = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter1 = default(Chilkat.Xml);
Currentlist.Clear();
list.Clear();
string[] tempC1 = null;
xNode1 = xml1.SearchForTag(xBeginSearchAfter1, "li");
while ((xNode1 != null))
{
Finaldata = xNode1.AccumulateTagContent("text", "/text");
Currentlist.Add(Finaldata);
// list.Add(Finaldata);
try
{
tempC1 = Regex.Split(Finaldata, " at ");
}
catch { }
if (tempC1 != null)
{
try
{
list.Add(tempC1[1]);
}
catch { }
}
xNode1 = xml1.SearchForTag(xBeginSearchAfter1, "li");
xBeginSearchAfter1 = xNode1;
}
if (Currentlist.Count > 0 || list.Count > 0)
{
try
{
titleCurrenttitle = Currentlist[0] != null ? Currentlist[0] : string.Empty;
titleCurrenttitle2 = Currentlist[1] != null ? Currentlist[1] : string.Empty;
titleCurrenttitle3 = Currentlist[2] != null ? Currentlist[2] : string.Empty;
titleCurrenttitle4 = Currentlist[3] != null ? Currentlist[3] : string.Empty;
}
catch { }
try
{
companyCurrenttitle1 = list[0] != null ? list[0] : string.Empty;
companyCurrenttitle2 = list[1] != null ? list[1] : string.Empty;
companyCurrenttitle3 = list[2] != null ? list[2] : string.Empty;
companyCurrenttitle4 = list[3] != null ? list[3] : string.Empty;
}
catch { }
}
}
}
catch { };
list.Clear();
#endregion
#region RegionForEDUCATION
try
{
if (html.Contains("summary-education"))
{
int FirstPointForEDUCATION = html.IndexOf("summary-education");
string FirstSubStringForEDUCATION = html.Substring(FirstPointForEDUCATION);
int SecondPointForEDUCATION = FirstSubStringForEDUCATION.IndexOf("<li>");
int ThirdPointForEDUCATION = FirstSubStringForEDUCATION.IndexOf("</ul>");
string SecondSubStringForEDUCATION = FirstSubStringForEDUCATION.Substring(SecondPointForEDUCATION, ThirdPointForEDUCATION - SecondPointForEDUCATION);
//string tempEDu = SecondSubStringForEDUCATION.Replace("<li>", string.Empty).Replace("</li>", string.Empty).Replace(" ", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Trim();
string temptg = SecondSubStringForEDUCATION.Replace("<li>", "");
string[] templis6t = temptg.Split('/');
education1 = templis6t[0].Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("<", string.Empty).Replace("span>", string.Empty).Replace(",", string.Empty).Trim();
education2 = templis6t[1].Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("li>", string.Empty).Replace("<", string.Empty).Replace("span>", string.Empty).Replace(",", string.Empty).Trim();
}
}
catch { };
list.Clear();
#endregion
string GroupPastJob = string.Empty;
string GroupEduction = string.Empty;
LDS_PastTitles = titlepast1 + ";" + titlepast3;
LDS_PastCompany = companypast1 + ";" + companypast3;
LDS_Education = education1 + ";" + education2;
LDS_CurrentTitle = titleCurrenttitle;
LDS_LoginID = SearchCriteria.LoginID; //"ProfileType" + "," + "UserProfileLink" + "," + "FirstName" + "," + "LastName" + "," + "HeadLineTitle" + "," + "CurrentTitle " + "," + "Company" + "," + "Connection" + "," + "Recommendations " + "," + "SkillAndExpertise " + "," + "Experience " + "," + " Education" + "," + "Groups" + "," + "UserEmail" + "," + "UserContactNumbe" + "," + "PastTitles" + "," + "PastCompany" + "," + "Loction" + "," + "Country" + "," + "titlepast3" + "," + "companypast3" + "," + "titlepast4" + "," + "companypast4" + ",";
string LDS_FinalData = LDS_ProfileType.Replace(",", ";") + "," + LDS_UserProfileLink.Replace(",", ";") + "," + LDS_FirstName.Replace(",", ";") + "," + LDS_LastName.Replace(",", ";") + "," + LDS_HeadLineTitle.Replace(",", ";") + "," + LDS_CurrentTitle.Replace(",", ";") + "," + LDS_CurrentCompany.Replace(",", ";") + "," + LDS_Connection.Replace(",", ";") + "," + LDS_Recommendations.Replace(",", ";") + "," + LDS_SkillAndExpertise.Replace(",", ";") + "," + LDS_Experience.Replace(",", ";") + "," + LDS_Education.Replace(",", ";") + "," + LDS_Groups.Replace(",", ";") + "," + LDS_UserEmail.Replace(",", ";") + "," + LDS_UserContactNumber.Replace(",", ";") + "," + LDS_PastTitles.Replace(",", ";") + "," + LDS_PastCompany.Replace(",", ";") + "," + LDS_Loction.Replace(",", ";") + "," + LDS_Country.Replace(",", ";") + "," + LDS_Industry.Replace(",", ";") + "," + LDS_Websites.Replace(",", ";") + "," + LDS_LoginID.Replace(",", ";") + ",";
if (LDS_FinalData.Contains("<strong class=\"highlight\"") || LDS_FinalData.Contains("<span class=\"full-name\"") || LDS_FinalData.Contains("<strong class=\"highlight\"") || LDS_FinalData.Contains("overview-connections\">"))
{
LDS_FinalData = LDS_FinalData.Replace("<span class=\"full-name\"", "").Replace("\n", "").Replace("<strong class=\"highlight\"", "").Replace("overview-connections\">", "").Replace("</strong>", "").Replace("<strong>", "");
}
if (!string.IsNullOrEmpty(LDS_FirstName) || !string.IsNullOrEmpty(LDS_FirstName))
{
Log("[ " + DateTime.Now + " ] => [ " + LDS_FinalData + " ]");
}
if (SearchCriteria.starter)
{
string tempFinalData = LDS_FinalData.Replace(";", "").Replace(LDS_UserProfileLink, "").Replace("Public", "").Replace(",", "").Replace(LDS_LoginID, "").Trim();
if (!string.IsNullOrEmpty(tempFinalData))
{
AppFileHelper.AddingLinkedInDataToCSVFile(LDS_FinalData, SearchCriteria.FileName);
//Log("Data Saved In CSV File With URL >>> " + LDS_UserProfileLink);
}
//if (!string.IsNullOrEmpty(LDS_FirstName) || !string.IsNullOrEmpty(LDS_FirstName))
//{
// AppFileHelper.AddingLinkedInDataToCSVFile(LDS_FinalData, SearchCriteria.FileName);
//}
}
}
catch (Exception ex) { };
}
}
}
public List<StructTweetIDs> NewKeywordStructDataSearchByPeople(string keyword)
{
try
{
BaseLib.GlobusRegex regx = new GlobusRegex();
int counter = 0;
lst_structTweetIDs = new List<StructTweetIDs>();
string res_Get_searchURL = string.Empty;
string searchURL = string.Empty;
string maxid = string.Empty;
string TweetId = string.Empty;
string text = string.Empty;
string ProfileName = string.Empty;
string Location = string.Empty;
string Bio = string.Empty;
string website = string.Empty;
string NoOfTweets = string.Empty;
string Followers = string.Empty;
string Followings = string.Empty;
if (keyword.Trim().Contains(" "))
{
keyword = keyword.Replace(" ", "+");
}
startAgain:
if (!RetweetFromUserName)
{
if (counter == 0)
{
//searchURL = "https://twitter.com/i/search/timeline?type=recent&src=typd&include_available_features=1&include_entities=1&max_id=0&q=" + keyword + "&composed_count=0&count=" + noOfRecords + "";
//searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime";
//29-4-2014 only for client it has been changed
searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&mode=users";
counter++;
}
else
{
//searchURL = "https://twitter.com/i/search/timeline?q=" + keyword + "&src=typd&f=realtime&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + "";
//29-4-2014 only for client it is changed
searchURL = "https://twitter.com/i/search/timeline?q=" + Uri.EscapeDataString(keyword) + "&src=typd&f=realtime&mode=users&include_available_features=1&include_entities=1&last_note_ts=0&oldest_unread_id=0&scroll_cursor=" + TweetId + "";
}
}
else
{
searchURL = "https://twitter.com/i/profiles/show/" + Uri.EscapeDataString(keyword) + "/timeline/with_replies?composed_count=0&count=" + RetweetExtractcount + "&include_available_features=1&include_entities=1";
}
try
{
res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
if (string.IsNullOrEmpty(res_Get_searchURL))
{
res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
}
try
{
//string sjss = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
string[] splitRes = Regex.Split(res_Get_searchURL, "refresh_cursor");
//splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
if (item.Contains("refresh_cursor"))
{
int startIndex = item.IndexOf("USER-");
string start = item.Substring(startIndex).Replace("data-user-id=\\\"", "");
int endIndex = start.IndexOf("\"");
string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "");
TweetId = end;
}
if (item.Contains("scroll_cursor"))
{
int startIndex = item.IndexOf("USER-");
string start = item.Substring(startIndex).Replace("data-user-id=\\\"", "");
int endIndex = start.IndexOf("\"");
string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "");
TweetId = end;
}
}
}
catch (Exception)
{
}
}
catch (Exception ex)
{
System.Threading.Thread.Sleep(2000);
res_Get_searchURL = globushttpHelper.getHtmlfromUrl(new Uri(searchURL), "", "");
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- res_Get_searchURL --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
// && !res_Get_searchURL.Contains("has_more_items\":false")
if (!string.IsNullOrEmpty(res_Get_searchURL))
{
//string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id"); //Regex.Split(res_Get_searchURL, "\"in_reply_to_status_id_str\"");
string[] splitRes = Regex.Split(res_Get_searchURL, "data-item-id");
splitRes = splitRes.Skip(1).ToArray();
foreach (string item in splitRes)
{
if (item.Contains("data-screen-name=") && !item.Contains("js-actionable-user js-profile-popup-actionable"))
{
//var avc = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(res_Get_searchURL);
//string DataHtml = (string)avc["items_html"];
}
else
{
//continue;
}
string modified_Item = "\"from_user\"" + item;
string id = "";
try
{
int startIndex = item.IndexOf("data-user-id=");
string start = item.Substring(startIndex).Replace("data-user-id=\\\"", "");
int endIndex = start.IndexOf("\\\"");
string end = start.Substring(0, endIndex).Replace("id_str", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", "");
id = end;
}
catch (Exception ex)
{
id = "null";
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- id -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
string from_user_id = "";
try
{
int startIndex = item.IndexOf("data-screen-name=\\\"");
string start = item.Substring(startIndex).Replace("data-screen-name=\\\"", "");
int endIndex = start.IndexOf("\\\"");
string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", "");
from_user_id = end;
}
catch (Exception ex)
{
from_user_id = "null";
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
string tweetUserid = string.Empty;
try
{
int startIndex = item.IndexOf("=\\\"");
string start = item.Substring(startIndex).Replace("=\\\"", "");
int endIndex = start.IndexOf("\\\"");
string end = start.Substring(0, endIndex).Replace("from_user_id\":", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("_str", "").Replace("user", "").Replace("}", "").Replace("]", "");
tweetUserid = end;
}
catch (Exception ex)
{
from_user_id = "null";
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user_id --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
StructTweetIDs structTweetIDs = new StructTweetIDs();
if (id != "null")
{
structTweetIDs.ID_Tweet = tweetUserid;
structTweetIDs.ID_Tweet_User = id;
structTweetIDs.username__Tweet_User = from_user_id;
structTweetIDs.wholeTweetMessage = text;
lst_structTweetIDs.Add(structTweetIDs);
Log("[ " + DateTime.Now + " ] => [ " + tweetUserid + " ]");
Log("[ " + DateTime.Now + " ] => [ " + id + " ]");
Log("[ " + DateTime.Now + " ] => [ " + from_user_id + " ]");
Log("-------------------------------------------------------------------------------------------------------------------------------");
if (!File.Exists(Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv"))
{
GlobusFileHelper.AppendStringToTextfileNewLine("USERID , USERNAME , PROFILE NAME , BIO , LOCATION , WEBSITE , NO OF TWEETS , FOLLOWERS , FOLLOWINGS", Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv");
}
//foreach (TwitterDataScrapper.StructTweetIDs item in data)
{
ChilkatHttpHelpr objChilkat = new ChilkatHttpHelpr();
GlobusHttpHelper HttpHelper = new GlobusHttpHelper();
string ProfilePageSource = HttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + from_user_id), "", "");
string Responce = ProfilePageSource;
#region Convert HTML to XML
string xHtml = objChilkat.ConvertHtmlToXml(Responce);
Chilkat.Xml xml = new Chilkat.Xml();
xml.LoadXml(xHtml);
Chilkat.Xml xNode = default(Chilkat.Xml);
Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml);
#endregion
int counterdata = 0;
xBeginSearchAfter = null;
string dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "h1", "class", "ProfileHeaderCard-name");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterdata == 0)
{
ProfileName = xNode.AccumulateTagContent("text", "script|style");
counterdata++;
}
else if (counterdata == 1)
{
website = xNode.AccumulateTagContent("text", "script|style");
counterdata++;
}
else
{
break;
}
// xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "profile-field");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "u-textUserColor");
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "ProfileHeaderCard-bio u-dir");//bio profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Bio = xNode.AccumulateTagContent("text", "script|style").Replace("'", "'").Replace(" ", string.Empty).Trim();
break;
}
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileHeaderCard-locationText u-dir");//location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
Location = xNode.AccumulateTagContent("text", "script|style");
break;
}
int counterData = 0;
xBeginSearchAfter = null;
dataDescription = string.Empty;
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-nav");//location profile-field");
while ((xNode != null))
{
xBeginSearchAfter = xNode;
if (counterData == 0)
{
// NoOfTweets = xml.SearchForAttribute(xBeginSearchAfter, "span", "class", "ProfileNav-value");
NoOfTweets = xNode.AccumulateTagContent("text", "script|style").Replace("Tweets", string.Empty).Replace(",", string.Empty).Replace("Tweet", string.Empty);
counterData++;
}
else if (counterData == 1)
{
Followings = xNode.AccumulateTagContent("text", "script|style").Replace(" Following", string.Empty).Replace(",", string.Empty).Replace("Following", string.Empty);
counterData++;
}
else if (counterData == 2)
{
Followers = xNode.AccumulateTagContent("text", "script|style").Replace("Followers", string.Empty).Replace(",", string.Empty).Replace("Follower", string.Empty);
counterData++;
}
else
{
break;
}
//xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "js-nav");
xNode = xml.SearchForAttribute(xBeginSearchAfter, "a", "class", "ProfileNav-stat ProfileNav-stat--link u-borderUserColor u-textCenter js-tooltip js-openSignupDialog js-nonNavigable u-textUserColor");
}
if (!string.IsNullOrEmpty(from_user_id) && tweetUserid != "null")
{
string Id_user = tweetUserid.Replace("}]", string.Empty).Trim();
Globals.lstScrapedUserIDs.Add(Id_user);
GlobusFileHelper.AppendStringToTextfileNewLine(id + "," + from_user_id + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location.Replace(",", "") + "," + website + "," + NoOfTweets.Replace(",", "").Replace("Tweets", "") + "," + Followers.Replace(",", "").Replace("Following", "") + "," + Followings.Replace(",", "").Replace("Followers", "").Replace("Follower", ""), Globals.Path_KeywordScrapedListData + "-" + keyword + ".csv");
GlobusFileHelper.AppendStringToTextfileNewLine(from_user_id, Globals.Path_KeywordScrapedListData + "-" + keyword + ".txt");
Log("[ " + DateTime.Now + " ] => [ " + from_user_id + "," + Id_user + "," + ProfileName + "," + Bio.Replace(",", "") + "," + Location + "," + website + "," + NoOfTweets + "," + Followers + "," + Followings + " ]");
}
}
}
lst_structTweetIDs = lst_structTweetIDs.Distinct().ToList();
if (lst_structTweetIDs.Count >= noOfRecords)
{
return lst_structTweetIDs;
}
}
if (lst_structTweetIDs.Count <= noOfRecords)
{
maxid = lst_structTweetIDs[lst_structTweetIDs.Count - 1].ID_Tweet;
if (res_Get_searchURL.Contains("has_moreitems\":false"))
{
return lst_structTweetIDs;
}
else
{
goto startAgain;
}
}
else
{
if (res_Get_searchURL.Contains("has_more_items\":false"))
{
return lst_structTweetIDs;
}
else
goto startAgain;
}
}
}
catch (Exception ex)
{
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwitterDataScrapper);
Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " --> " + ex.Message, Globals.Path_TwtErrorLogs);
}
return lst_structTweetIDs;
}
public static void CreateProfileXml()
{
DBConnect proc = new DBConnect();
DataTable rs3;
DataTable rs2;
DataTable rs;
int i = 0;
int j = 0;
int k = 0;
rs = proc.getAllProfil();
rs2 = proc.getAllPoste();
rs3 = proc.getAllTeam();
// Create the root node of a new XML document.
Chilkat.Xml xml = new Chilkat.Xml();
// This is going to be a collection of contacts.
xml.Tag = "profiles";
// Create a child node to hold a contact record.
// Our "xml" object will now reference the new node.
while (i < rs.Rows.Count)
{
xml = xml.NewChild("profil", "");
// Create a "company" record. NewChild2 does not return the child node that is created
xml.NewChild2("id", Convert.ToString(rs.Rows[i]["idProfil"]));
xml.NewChild2("nom", rs.Rows[i]["nom"].ToString()+", "+rs.Rows[i]["prenom"].ToString());
xml.NewChild2("email", rs.Rows[i]["email"].ToString());
xml.NewChild2("phone", rs.Rows[i]["phoneNumber"].ToString());
xml.NewChild2("seniority", Convert.ToString(rs.Rows[i]["seniority"]));
// Move up the tree and create another contact record.
// GetParent2 updates our internal reference to the parent.
xml.GetParent2();
i++;
}
// We can always get back to the root by calling GetRoot (or GetRoot2)
xml.GetRoot2();
// Add the tag that specifies our character encoding.
xml.Encoding = "utf-8";
// Save the document.
xml.SaveXml("profiles.xml");
//GÉNÉRATION DE POSTES.XML
xml = new Chilkat.Xml();
xml.Tag = "postes";
while (j < rs2.Rows.Count)
{
xml = xml.NewChild("poste", "");
xml.NewChild2("id", Convert.ToString(rs2.Rows[j]["idPoste"]));
xml.NewChild2("nom", rs2.Rows[j]["nom"].ToString());
xml.NewChild2("description", rs2.Rows[j]["description"].ToString());
xml.GetParent2();
j++;
}
// We can always get back to the root by calling GetRoot (or GetRoot2)
xml.GetRoot2();
// Add the tag that specifies our character encoding.
xml.Encoding = "utf-8";
// Save the document.
xml.SaveXml("postes.xml");
//GÉNÉRATION DE POSTES.XML
xml = new Chilkat.Xml();
xml.Tag = "teams";
while (k < rs3.Rows.Count)
{
xml = xml.NewChild("team", "");
xml.NewChild2("id", Convert.ToString(rs3.Rows[k]["idTeam"]));
xml.NewChild2("nom", rs3.Rows[k]["nom"].ToString());
xml.NewChild2("description", rs3.Rows[k]["description"].ToString());
xml.GetParent2();
k++;
}
// We can always get back to the root by calling GetRoot (or GetRoot2)
xml.GetRoot2();
// Add the tag that specifies our character encoding.
xml.Encoding = "utf-8";
// Save the document.
xml.SaveXml("teams.xml");
}
