public bool CrawlingLinkedInPage(string Url, ref GlobusHttpHelper HttpHelper,string FileName)
{
//Url = "http://www.linkedin.com/profile/view?id=44952194&authType=OUT_OF_NETWORK&authToken=DYBR&locale=en_US&srchid=3817933251416230963594&srchindex=2&srchtotal=1949&trk=vsrp_people_res_name&trkInfo=VSRPsearchId%3A3817933251416230963594%2CVSRPtargetId%3A44952194%2CVSRPcmpt%3Aprimary";
#region Data Initialization
string GroupMemId = string.Empty;
string Industry = string.Empty;
string URLprofile = string.Empty;
string firstname = string.Empty;
string lastname = string.Empty;
string location = string.Empty;
string country = string.Empty;
string postal = string.Empty;
string phone = string.Empty;
string USERemail = string.Empty;
string code = string.Empty;
string education1 = string.Empty;
string education2 = string.Empty;
string titlecurrent = string.Empty;
string companycurrent = string.Empty;
string CurrentCompUrl = string.Empty;
string CurrentCompSite = string.Empty;
string titlepast1 = string.Empty;
string companypast1 = string.Empty;
string titlepast2 = string.Empty;
string html = string.Empty;
string companypast2 = string.Empty;
string titlepast3 = string.Empty;
string companypast3 = string.Empty;
string titlepast4 = string.Empty;
string companypast4 = string.Empty;
string Recommendations = string.Empty;
string Connection = string.Empty;
string Designation = string.Empty;
string Website = string.Empty;
string Contactsettings = string.Empty;
string recomandation = string.Empty;
string titleCurrenttitle = string.Empty;
string titleCurrenttitle2 = string.Empty;
string titleCurrenttitle3 = string.Empty;
string titleCurrenttitle4 = string.Empty;
string Skill = string.Empty;
string TypeOfProfile = "Public";
List<string> EducationList = new List<string>();
string Finaldata = string.Empty;
string EducationCollection = string.Empty;
List<string> checkerlst = new List<string>();
List<string> checkgrplist = new List<string>();
string groupscollectin = string.Empty;
string strFamilyName = string.Empty;
string LDS_LoginID = string.Empty;
string LDS_Websites = string.Empty;
string LDS_UserProfileLink = string.Empty;
string LDS_CurrentTitle = string.Empty;
string LDS_Experience = string.Empty;
string LDS_UserContact = string.Empty;
string LDS_PastTitles = string.Empty;
string LDS_BackGround_Summary = string.Empty;
string LDS_Desc_AllComp = string.Empty;
string Company = string.Empty;
List<string> lstpasttitle = new List<string>();
List<string> checkpasttitle = new List<string>();
string DeegreeConn = string.Empty;
string AccountType = string.Empty;
bool CheckEmployeeScraper = false;
string fileName = string.Empty;
bool CampaignScraper = false;
#endregion
#region GetRequest
if (Url.Contains("CompanyEmployeeScraper"))
{
try
{
Url = Url.Replace("CompanyEmployeeScraper", string.Empty);
CheckEmployeeScraper = true;
}
catch
{ }
}
if (Url.Contains("CampaignScraper"))
{
try
{
string[] Url_Split = Regex.Split(Url, "CampaignScraper");
Url = Url_Split[0];
fileName = Url_Split[1];
CampaignScraper = true;
}
catch
{ }
}
string stringSource = HttpHelper.getHtmlfromUrl(new Uri(Url.Replace("http", "https")));
#endregion
#region GroupMemId
try
{
string[] gid = Url.Split('&');
GroupMemId = gid[0].Replace("http://www.linkedin.com/profile/view?id=", string.Empty);
}
catch { }
#endregion
#region Name
try
{
try
{
strFamilyName = stringSource.Substring(stringSource.IndexOf("\"See Full Name\",\"i18n_Edit\":\"Edit\",\"fmt__full_name\":\""), (stringSource.IndexOf("i18n__expand_your_network_to_see_more", stringSource.IndexOf("\"See Full Name\",\"i18n_Edit\":\"Edit\",\"fmt__full_name\":\"")) - stringSource.IndexOf("\"See Full Name\",\"i18n_Edit\":\"Edit\",\"fmt__full_name\":\""))).Replace("\"See Full Name\",\"i18n_Edit\":\"Edit\",\"fmt__full_name\":\"", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim();
}
catch
{
try
{
strFamilyName = stringSource.Substring(stringSource.IndexOf("fmt__full_name\":"), (stringSource.IndexOf(",", stringSource.IndexOf("fmt__full_name\":")) - stringSource.IndexOf("fmt__full_name\":"))).Replace("fmt__full_name\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim();
}
catch { }
}
if (string.IsNullOrEmpty(strFamilyName))
{
try
{
strFamilyName = stringSource.Substring(stringSource.IndexOf("<span class=\"full-name\">"), (stringSource.IndexOf("</span><span></span></span></h1></div></div><div id=\"headline-container\" data-li-template=\"headline\">", stringSource.IndexOf("</span><span></span></span></h1></div></div><div id=\"headline-container\" data-li-template=\"headline\">")) - stringSource.IndexOf("<span class=\"full-name\">"))).Replace("<span class=\"full-name\">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim();
}
catch
{ }
}
if (string.IsNullOrEmpty(strFamilyName))
{
try
{
int StartIndex = stringSource.IndexOf("profileFullName");
string Start = stringSource.Substring(StartIndex).Replace("profileFullName", string.Empty);
int EndIndex = Start.IndexOf("slideData");
string End = Start.Substring(0, EndIndex).Replace(":", string.Empty).Replace("'",string.Empty).Replace(",",string.Empty).Trim();
strFamilyName = End.Trim();
}
catch
{ }
}
}
catch { }
if (string.IsNullOrEmpty(strFamilyName))
{
try
{
int StartIndex = stringSource.IndexOf("</script><title>");
string Start = stringSource.Substring(StartIndex).Replace("</script><title>", string.Empty);
int EndIndex = Start.IndexOf("| LinkedIn</title>");
string End = Start.Substring(0, EndIndex).Replace(":", string.Empty).Replace("'", string.Empty).Replace(",", string.Empty).Trim();
strFamilyName = End.Trim();
}
catch
{ }
}
//
#endregion
#region Namesplitation
string[] NameArr = new string[5];
if (strFamilyName.Contains(" "))
{
try
{
NameArr = Regex.Split(strFamilyName, " ");
}
catch { }
}
#endregion
#region FirstName
try
{
firstname = NameArr[0];
}
catch { }
#endregion
#region LastName
try
{
lastname = NameArr[1];
}
catch { }
try
{
if (NameArr.Count() == 3)
{
try
{
lastname = NameArr[1] + " " + NameArr[2];
}
catch { }
}
if (lastname.Contains("}]"))
{
#region Name
try
{
try
{
strFamilyName = stringSource.Substring(stringSource.IndexOf("<span class=\"n fn\">")).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim();
}
catch
{
try
{
strFamilyName = stringSource.Substring(stringSource.IndexOf("fmt__full_name\":"), (stringSource.IndexOf(",", stringSource.IndexOf("fmt__full_name\":")) - stringSource.IndexOf("fmt__full_name\":"))).Replace("fmt__full_name\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim();
}
catch { }
}
}
catch { }
#endregion
}
}
catch { }
#endregion
#region Company
try
{
try
{
try
{
//soft engg at TCSi18n__LocationLocationi18n__Linkedin_memberLinkedIn Member
//Company = stringSource.Substring(stringSource.IndexOf("\"memberHeadline"), (stringSource.IndexOf("memberID", stringSource.IndexOf("\"memberHeadline")) - stringSource.IndexOf("\"memberHeadline"))).Replace("\"memberHeadline", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace(" ", string.Empty).Replace("visibletrue", string.Empty).Replace("isLNLedtrue",string.Empty).Replace("isPortfoliofalse",string.Empty).Replace("i18n__Location",string.Empty).Replace("Locationi18n__Linkedin_member",string.Empty).Replace("u002d","-").Replace("LinkedIn Member",string.Empty).Replace("--Location","--").Trim();
Company = stringSource.Substring(stringSource.IndexOf("\"memberHeadline"), (stringSource.IndexOf("memberID", stringSource.IndexOf("\"memberHeadline")) - stringSource.IndexOf("\"memberHeadline"))).Replace("\"memberHeadline", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace(" ", string.Empty).Replace("visibletrue", string.Empty).Replace("isLNLedtrue", string.Empty).Replace("isPortfoliofalse", string.Empty).Replace("i18n__Location", string.Empty).Replace("Locationi18n__Linkedin_member", string.Empty).Replace("u002d", "-").Replace("LinkedIn Member", string.Empty).Replace("--Location", "--").ToString().Trim();
if (Company.Contains("#Name?"))
{
Company = "--";
}
if (Company.Contains("i18n"))
{
Company = Regex.Split(Company, "i18n")[0];
}
}
catch
{
}
if (string.IsNullOrEmpty(Company))
{
try
{
Company = stringSource.Substring(stringSource.IndexOf("memberHeadline\":"), (stringSource.IndexOf(",", stringSource.IndexOf("memberHeadline\":")) - stringSource.IndexOf("memberHeadline\":"))).Replace("memberHeadline\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace(" ", string.Empty).Replace(":", "").Replace("visibletrue", string.Empty).Replace("&dsh;", "").Replace("&", "").Replace(";", "").Replace("isLNLedtrue", string.Empty).Replace("isPortfoliofalse", string.Empty).Trim();
}
catch { }
}
if (string.IsNullOrEmpty(Company))
{
try
{
Company = stringSource.Substring(stringSource.IndexOf("class=\"title \">"), (stringSource.IndexOf("</p></div></div><div class=\"demographic-info adr editable-item\" id=\"demographics\">", stringSource.IndexOf("</p></div></div><div class=\"demographic-info adr editable-item\" id=\"demographics\">")) - stringSource.IndexOf("class=\"title \">"))).Replace("class=\"title \">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace("<strong class=highlight>", string.Empty).Replace("</strong>", string.Empty).Trim();
}
catch { }
}
string[] strdesigandcompany = new string[4];
if (Company.Contains(" at "))
{
try
{
strdesigandcompany = Regex.Split(Company, " at ");
}
catch { }
#region Title
try
{
titlecurrent = strdesigandcompany[0];
}
catch { }
#endregion
#region Current Company
try
{
companycurrent = strdesigandcompany[1];
}
catch { }
#endregion
}
}
catch { }
#region Current Company Site
try
{
try
{
CurrentCompUrl = stringSource.Substring(stringSource.IndexOf("/companies"), (stringSource.IndexOf("/companies", stringSource.IndexOf("?miniprofile")) - stringSource.IndexOf("?miniprofile"))).Replace("?miniprofi", string.Empty).ToString().Trim();
CurrentCompUrl = "https://www.linkedin.com" + CurrentCompUrl;
}
catch { }
string CompanyUrl = HttpHelper.getHtmlfromUrl1(new Uri(CurrentCompUrl));
try
{
CurrentCompSite = CompanyUrl.Substring(CompanyUrl.IndexOf("<dt>Website</dt>"), (CompanyUrl.IndexOf("</dd>", CompanyUrl.IndexOf("<dt>Website</dt>")) - CompanyUrl.IndexOf("<dt>Website</dt>"))).Replace("<dt>Website</dt>", string.Empty).Replace("<dd>", string.Empty).Trim();
}
catch { }
try
{
CurrentCompSite = CompanyUrl.Substring(CompanyUrl.IndexOf("<h4>Website</h4>"), (CompanyUrl.IndexOf("</p>", CompanyUrl.IndexOf("<h4>Website</h4>")) - CompanyUrl.IndexOf("<h4>Website</h4>"))).Replace("<h4>Website</h4>", string.Empty).Replace("<p>", string.Empty).Trim();
if (CurrentCompSite.Contains("a href="))
{
try
{
string[] websArr = Regex.Split(CurrentCompSite, ">");
CurrentCompSite = websArr[1].Replace("</a", string.Empty).Replace("\n", string.Empty).Trim(); ;
}
catch { }
}
}
catch { }
}
catch { }
#endregion
#region PastCompany
string[] companylist = Regex.Split(stringSource, "companyName\"");
if (companylist.Count() == 1)
{
companylist = Regex.Split(stringSource, "company-name");
}
if (companylist.Count() == 1)
{
//companylist = Regex.Split(stringSource, "Companies");
}
string AllComapny = string.Empty;
string Companyname = string.Empty;
if (!stringSource.Contains("company-name") && companylist.Count() > 1)
{
foreach (string item in companylist)
{
try
{
if (!item.Contains("<!DOCTYPE html>"))
{
Companyname = item.Substring(item.IndexOf(":"), (item.IndexOf(",", item.IndexOf(":")) - item.IndexOf(":"))).Replace(":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace("}", string.Empty).Replace("]", string.Empty).Trim();
string items = item;
checkerlst.Add(Companyname);
checkerlst = checkerlst.Distinct().ToList();
}
}
catch { }
}
}
else
{
foreach (string item in companylist)
{
try
{
if (!item.Contains("<!DOCTYPE html>"))
{
Companyname = item.Substring(item.IndexOf(">"), (item.IndexOf("<", item.IndexOf(">")) - item.IndexOf(">"))).Replace(">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace("}", string.Empty).Replace("]", string.Empty).Trim();
string items = item;
if (!string.IsNullOrEmpty(Companyname))
{
checkerlst.Add(Companyname);
checkerlst = checkerlst.Distinct().ToList();
}
}
}
catch { }
}
}
AllComapny = string.Empty;
foreach (string item1 in checkerlst)
{
if (string.IsNullOrEmpty(AllComapny))
{
AllComapny = item1.Replace("}", "").Replace("]", "").Replace("&", "&");
}
else
{
AllComapny = AllComapny + " : " + item1.Replace("}", "").Replace("]", "").Replace("&", "&");
}
}
#endregion
#endregion Company
#region Company Descripription
try
{
string[] str_CompanyDesc = Regex.Split(stringSource, "showSummarySection");
foreach (string item in str_CompanyDesc)
{
try
{
string Current_Company = string.Empty;
if (!item.Contains("<!DOCTYPE html>"))
{
int startindex = item.IndexOf("specialties\":\"");
if (startindex > 0)
{
try
{
string start = item.Substring(startindex).Replace("specialties\":", "");
int endindex = start.IndexOf("\",\"associatedWith\"");
string end = start.Substring(0, endindex);
Current_Company = end.Replace(",\"specialties_lb\":", string.Empty).Replace("\"", string.Empty).Replace("summary_lb", "Summary").Replace("•", ";").Replace("<br>", string.Empty).Replace("\\n", string.Empty).Replace("\"u002", "-");
LDS_BackGround_Summary = Current_Company;
}
catch { }
}
}
if (!item.Contains("<!DOCTYPE html>"))
{
int startindex = item.IndexOf("\"summary_lb\"");
if (startindex > 0)
{
try
{
string start = item.Substring(startindex).Replace("\"summary_lb\"", "");
int endindex = start.IndexOf("\",\"associatedWith\"");
string end = start.Substring(0, endindex);
Current_Company = end.Replace(",\"specialties_lb\":", string.Empty).Replace("<br>", string.Empty).Replace("\n", string.Empty).Replace("\"", string.Empty).Replace("summary_lb", "Summary").Replace(",", ";").Replace("u002", "-").Replace("•", string.Empty).Replace(":", string.Empty);
LDS_BackGround_Summary = Current_Company;
}
catch { }
}
}
}
catch { }
}
if (string.IsNullOrEmpty(LDS_BackGround_Summary))
{
try
{
LDS_BackGround_Summary = HttpHelper.GetDataWithTagValueByTagAndAttributeNameWithId(stringSource, "div", "summary-item-view");
LDS_BackGround_Summary = Regex.Replace(LDS_BackGround_Summary, "<.*?>", string.Empty).Replace(",", "").Replace("\n", "").Replace("<![CDATA[", "").Trim();
}
catch { }
}
}
catch { }
#endregion
#region Education
try
{
string[] str_UniversityName = Regex.Split(stringSource, "link__school_name");
foreach (string item in str_UniversityName)
{
try
{
string School = string.Empty;
string Degree = string.Empty;
string SessionEnd = string.Empty;
string SessionStart = string.Empty;
string Education = string.Empty;
if (stringSource.Contains("link__school_name"))
{
if (!item.Contains("<!DOCTYPE html>"))
{
try
{
try
{
int startindex = item.IndexOf("fmt__school_highlight");
string start = item.Substring(startindex).Replace("fmt__school_highlight", "");
int endindex = start.IndexOf(",");
School = start.Substring(0, endindex).Replace("\\u002d", string.Empty).Replace(":", string.Empty).Replace("\"", string.Empty).Replace("_highlight", string.Empty);
}
catch { }
try
{
int startindex1 = item.IndexOf("degree");
string start1 = item.Substring(startindex1).Replace("degree", "");
int endindex1 = start1.IndexOf(",");
Degree = start1.Substring(0, endindex1).Replace("\\u002d", string.Empty).Replace(":", string.Empty).Replace("\"", string.Empty).Replace("_highlight", string.Empty);
}
catch { }
try
{
int startindex2 = item.IndexOf("enddate_my");
string start2 = item.Substring(startindex2).Replace("enddate_my", "");
int endindex2 = start2.IndexOf(",");
SessionEnd = start2.Substring(0, endindex2).Replace("\\u002d", string.Empty).Replace(":", string.Empty).Replace("\"", string.Empty).Replace("_highlight", string.Empty);
}
catch { }
try
{
int startindex3 = item.IndexOf("startdate_my");
string start3 = item.Substring(startindex3).Replace("startdate_my", "");
int endindex3 = start3.IndexOf(",");
SessionStart = start3.Substring(0, endindex3).Replace("\\u002d", string.Empty).Replace(":", string.Empty).Replace("\"", string.Empty);
}
catch { }
if (SessionStart == string.Empty && SessionEnd == string.Empty)
{
Education = " [" + School + "] Degree: " + Degree;
}
else
{
Education = " [" + School + "] Degree: " + Degree + " Session: " + SessionStart + "-" + SessionEnd;
}
//University = item.Substring(item.IndexOf(":"), (item.IndexOf(",", item.IndexOf(":")) - item.IndexOf(":"))).Replace(":", string.Empty).Replace("\\u002d", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim();
}
catch { }
EducationList.Add(Education);
}
}
else
{
str_UniversityName = Regex.Split(stringSource, "<div class=\"education");
foreach (string tempItem in str_UniversityName)
{
try
{
if (!tempItem.Contains("<!DOCTYPE html>"))
{
List<string> lstSchool = HttpHelper.GetTextDataByTagAndAttributeName(tempItem, "h4", "summary fn org");
List<string> lstDegree = HttpHelper.GetTextDataByTagAndAttributeName(tempItem, "span", "degree");
List<string> lstSession = HttpHelper.GetTextDataByTagAndAttributeName(tempItem, "span", "education-date");
if (lstSession.Count == 0)
{
Education = " [" + lstSchool[0] + "] Degree: " + lstDegree[0];
}
else
{
Education = " [" + lstSchool[0] + "] Degree: " + lstDegree[0] + " Session: " + lstSession[0].Replace("–", "-").Replace(",", "").Trim();
}
EducationList.Add(Education);
}
}
catch { }
}
}
}
catch { }
}
EducationList = EducationList.Distinct().ToList();
foreach (string item in EducationList)
{
if (string.IsNullOrEmpty(EducationCollection))
{
EducationCollection = item.Replace("}", "").Replace("]", "").Replace("&", "&");
}
else
{
EducationCollection = EducationCollection + " - " + item.Replace("}", "").Replace("]", "").Replace("&", "&");
}
}
// string University1 = stringSource.Substring(stringSource.IndexOf("schoolName\":"), (stringSource.IndexOf(",", stringSource.IndexOf("schoolName\":")) - stringSource.IndexOf("schoolName\":"))).Replace("schoolName\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim();
}
catch { }
#endregion Education
#region Email
try
{
if (stringSource.Contains("mailto:"))
{
string[] str_Email = Regex.Split(stringSource, "mailto:");
USERemail = stringSource.Substring(stringSource.IndexOf("mailto:"), (stringSource.IndexOf(">", stringSource.IndexOf("mailto:")) - stringSource.IndexOf("mailto:"))).Replace("mailto:", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim();
}
else
{
string[] str_Email = Regex.Split(stringSource, "email\"");
USERemail = stringSource.Substring(stringSource.IndexOf("[{\"email\":"), (stringSource.IndexOf("}]", stringSource.IndexOf("[{\"email\":")) - stringSource.IndexOf("[{\"email\":"))).Replace("[{\"email\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim();
}
}
catch (Exception ex)
{
}
#endregion Email
#region UserContact
try
{
if (stringSource.Contains("<div id=\"phone-view\">"))
{
//string[] str_Contact = Regex.Split(stringSource, "<div id=\"phone-view\">");
LDS_UserContact = stringSource.Substring(stringSource.IndexOf("<div id=\"phone-view\">"), (stringSource.IndexOf("</li>", stringSource.IndexOf("<div id=\"phone-view\">")) - stringSource.IndexOf("<div id=\"phone-view\">"))).Replace("<div id=\"phone-view\">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace("<ul><li>", string.Empty).Replace(" ", "").Trim();
}
else
{
//string[] str_Email = Regex.Split(stringSource, "<div id=\"phone-view\">");
LDS_UserContact = stringSource.Substring(stringSource.IndexOf("<div id=\"phone-view\">"), (stringSource.IndexOf("</li>", stringSource.IndexOf("<div id=\"phone-view\">")) - stringSource.IndexOf("<div id=\"phone-view\">"))).Replace("<div id=\"phone-view\">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace("<ul><li>", string.Empty).Replace(" ", "").Trim();
}
}
catch (Exception ex)
{
}
#endregion Email
#region Website
try
{
Website = stringSource.Substring(stringSource.IndexOf("[{\"URL\":"), (stringSource.IndexOf(",", stringSource.IndexOf("[{\"URL\":")) - stringSource.IndexOf("[{\"URL\":"))).Replace("[{\"URL\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace("}", string.Empty).Replace("]", string.Empty).Trim();
}
catch { }
if (string.IsNullOrEmpty(Website))
{
try
{
Website = HttpHelper.GetDataWithTagValueByTagAndAttributeNameWithId(stringSource, "div", "website-view");
Website = Regex.Replace(Website, "<[^>]*>", String.Empty).Replace("\n", "").Replace("\r", "").Replace(" ", " ").Trim();
Website = Regex.Replace(Website, @"\s+", " ").Replace(",", " ").Trim();
}
catch { }
}
#endregion Website
#region location
try
{
//location = stringSource.Substring(stringSource.IndexOf("Country\",\"fmt__location\":"), (stringSource.IndexOf("i18n_no_location_matches", stringSource.IndexOf("Country\",\"fmt__location\":")) - stringSource.IndexOf("Country\",\"fmt__location\":"))).Replace("Country\",\"fmt__location\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim();
int startindex = stringSource.IndexOf("fmt_location");
string start = stringSource.Substring(startindex).Replace("fmt_location\":\"", "");
int endindex = start.IndexOf("\"");
string end = start.Substring(0, endindex).Replace("\u002d", string.Empty);
location = end;
}
catch (Exception ex)
{
}
if (string.IsNullOrEmpty(location))
{
try
{
List<string> lstLocation = HttpHelper.GetTextDataByTagAndAttributeName(stringSource, "span", "locality");
if (lstLocation.Count > 0)
{
location = lstLocation[lstLocation.Count - 1].Trim();
}
}
catch { }
}
#endregion location
#region Country
try
{
int startindex = stringSource.IndexOf("\"geo_region\":");
if (startindex > 0)
{
string start = stringSource.Substring(startindex).Replace("\"geo_region\":", "");
int endindex = start.IndexOf("\"i18n_geo_region\":\"Location\"");
string end = start.Substring(0, endindex);
country = end;
string[] array = Regex.Split(end, "\"name\":\"");
array = array.Skip(1).ToArray();
foreach (string item in array)
{
try
{
int startindex1 = item.IndexOf("\",\"");
string strat1 = item.Substring(0, startindex1);
country = strat1;
break;
}
catch (Exception ex)
{
}
}
}
}
catch (Exception ex)
{
}
if (country == string.Empty)
{
try
{
string[] countLocation = location.Split(',');
if (countLocation.Count() == 2)
{
country = location.Split(',')[1];
}
else if (countLocation.Count() == 3)
{
country = location.Split(',')[2];
}
}
catch { }
}
#endregion
#region Industry
try
{
//Industry = stringSource.Substring(stringSource.IndexOf("fmt__industry_highlight\":"), (stringSource.IndexOf(",", stringSource.IndexOf("fmt__industry_highlight\":")) - stringSource.IndexOf("fmt__industry_highlight\":"))).Replace("fmt__industry_highlight\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim();
int startindex = stringSource.IndexOf("\"industry_highlight\":\"");
if (startindex > 0)
{
string start = stringSource.Substring(startindex).Replace("\"industry_highlight\":\"", "");
int endindex = start.IndexOf("\",");
string end = start.Substring(0, endindex).Replace("\"", string.Empty).Replace("</strong>", string.Empty).Replace("&", "&");
if (end.Contains("strong class"))
{
Industry = end.Split('>')[1];
}
else
{
Industry = end;
}
}
}
catch (Exception ex)
{
}
if (string.IsNullOrEmpty(Industry))
{
try
{
List<string> lstIndustry = HttpHelper.GetTextDataByTagAndAttributeName(stringSource, "dd", "industry");
if (lstIndustry.Count > 0)
{
Industry = lstIndustry[0].Replace(",", ":").Trim();
}
}
catch { }
}
#endregion Industry
#region Connection
try
{
//Connection = stringSource.Substring(stringSource.IndexOf("_count_string\":"), (stringSource.IndexOf(",", stringSource.IndexOf("_count_string\":")) - stringSource.IndexOf("_count_string\":"))).Replace("_count_string\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim();
int startindex = stringSource.IndexOf("\"numberOfConnections\":");
if (startindex > 0)
{
string start = stringSource.Substring(startindex).Replace("\"numberOfConnections\":", "");
int endindex = start.IndexOf(",");
string end = start.Substring(0, endindex).Replace("&", "&").Replace("}", string.Empty);
Connection = end;
}
}
catch (Exception ex)
{
}
if (string.IsNullOrEmpty(Connection))
{
try
{
List<string> lstConnection = HttpHelper.GetTextDataByTagAndAttributeName(stringSource, "div", "member-connections");
if (lstConnection.Count > 0)
{
Connection = lstConnection[0].Replace(",", ":").Trim();
}
}
catch { }
}
#endregion Connection
#region Recommendation
try
{
string RecomnedUrl = string.Empty;
try
{
int startindex = stringSource.IndexOf("endorsements?id=");
string start = stringSource.Substring(startindex);
int endIndex = start.IndexOf("\"mem_pic\":");
if (endIndex < 0)
{
endIndex = start.IndexOf(">");
}
RecomnedUrl = (start.Substring(0, endIndex).Replace(",", string.Empty).Replace("\"", string.Empty).Replace(":", string.Empty));
}
catch { }
string PageSource = HttpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/profile/profile-v2-" + RecomnedUrl + ""));
string[] arrayRecommendedName = Regex.Split(PageSource, "headline");
if (arrayRecommendedName.Count() == 1)
{
arrayRecommendedName = Regex.Split(PageSource, "fmt__recommendeeFullName");
}
List<string> ListRecommendationName = new List<string>();
foreach (var itemRecomName in arrayRecommendedName)
{
try
{
if (!itemRecomName.Contains("Endorsements"))
{
string Heading = string.Empty;
string Name = string.Empty;
try
{
int startindex = itemRecomName.IndexOf(":");
string start = itemRecomName.Substring(startindex);
int endIndex = start.IndexOf("\",");
Heading = (start.Substring(0, endIndex).Replace("\"", string.Empty).Replace(":", string.Empty).Replace(",", ";"));
}
catch { }
try
{
int startindex1 = itemRecomName.IndexOf("fmt__referrerfullName");
string start1 = itemRecomName.Substring(startindex1);
int endIndex1 = start1.IndexOf("\",");
Name = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace("fmt__referrerfullName", string.Empty).Replace(":", string.Empty));
}
catch { }
if (Name == string.Empty)
{
int startindex1 = itemRecomName.IndexOf("recommenderTitle\":");
string start1 = itemRecomName.Substring(startindex1);
int endIndex1 = start1.IndexOf("\",");
Name = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace("recommenderTitle", string.Empty).Replace(":", string.Empty).Replace(",", string.Empty));
}
ListRecommendationName.Add(Name + " : " + Heading);
}
}
catch { }
}
foreach (var item in ListRecommendationName)
{
if (recomandation == string.Empty)
{
recomandation = item;
}
else
{
recomandation += " - " + item;
}
}
}
catch { }
#endregion
#region Following
#endregion
#region Experience
if (LDS_Experience == string.Empty)
{
try
{
string[] array = Regex.Split(stringSource, "title_highlight");
string exp = string.Empty;
string comp = string.Empty;
List<string> ListExperince = new List<string>();
string SelItem = string.Empty;
if (stringSource.Contains("title_highlight"))
{
foreach (var itemGrps in array)
{
try
{
if (itemGrps.Contains("title_pivot") && !itemGrps.Contains("<!DOCTYPE html")) //">Join
{
try
{
int startindex = itemGrps.IndexOf("\":\"");
string start = itemGrps.Substring(startindex);
int endIndex = start.IndexOf(",");
exp = (start.Substring(0, endIndex).Replace("\"", string.Empty).Replace(":", string.Empty).Replace("&", "&").Replace(";", string.Empty).Replace("\\u002d", "-").Replace("name:", string.Empty));
}
catch { }
if ((exp.Contains("strong class")) || (exp.Contains("highlight")) || (string.IsNullOrEmpty(exp)))
{
try
{
int startindex1 = itemGrps.IndexOf("\"title\":");
string start1 = itemGrps.Substring(startindex1).Replace("\"title\":", string.Empty);
int endIndex1 = start1.IndexOf(",");
exp = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace(":", string.Empty).Replace("&", "&").Replace(";", string.Empty).Replace("\\u002d", "-").Replace("name:", string.Empty));
}
catch
{ }
}
try
{
int startindex1 = itemGrps.IndexOf("companyName");
string start1 = itemGrps.Substring(startindex1);
int endIndex1 = start1.IndexOf(",");
comp = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace("companyName", string.Empty).Replace(":", string.Empty).Replace(";", string.Empty).Replace("\\u002d", "-").Replace("name:", string.Empty));
}
catch { }
if (titlecurrent == string.Empty)
{
titlecurrent = exp;
}
if (companycurrent == string.Empty)
{
companycurrent = comp;
}
ListExperince.Add(exp + ":" + comp);
}
}
catch { }
}
}
else
{
array = Regex.Split(stringSource, "<header>");
foreach (string tempItem in array)
{
try
{
if (!tempItem.Contains("<!DOCTYPE html>"))
{
List<string> lstExp = objChilkat.GetDataTag(tempItem, "h4");
List<string> lstComp = objChilkat.GetDataTag(tempItem, "h5");
if (lstExp.Count > 0)
{
exp = lstExp[0];
}
if (lstComp.Count > 0)
{
comp = lstComp[0];
if (string.IsNullOrEmpty(comp))
{
comp = lstComp[1];
}
}
if (titlecurrent == string.Empty)
{
titlecurrent = lstExp[0];
}
if (companycurrent == string.Empty)
{
companycurrent = lstComp[0];
}
ListExperince.Add(exp + ":" + comp);
}
}
catch { }
}
}
foreach (var item in ListExperince)
{
if (LDS_Experience == string.Empty)
{
LDS_Experience = item;
}
else
{
LDS_Experience += " - " + item;
}
}
}
catch { }
try
{
if (string.IsNullOrEmpty(titlecurrent))
{
int StartIndex = stringSource.IndexOf("trk=prof-0-ovw-curr_pos\">");
string Start = stringSource.Substring(StartIndex).Replace("trk=prof-0-ovw-curr_pos\">", string.Empty);
int EndIndex = Start.IndexOf("</a>");
string End = Start.Substring(0, EndIndex).Replace("</a>", string.Empty);
titlecurrent = End.Trim();
}
}
catch
{ }
}
#endregion
#region Group
try
{
string GroupUrl = string.Empty;
try
{
int startindex = stringSource.IndexOf("templateId\":\"profile_v2_connections");
string start = stringSource.Substring(startindex);
//int endIndex = start.IndexOf("vsrp_people_res_name");
int endIndex = start.IndexOf("}");
GroupUrl = (start.Substring(0, endIndex).Replace(",", string.Empty).Replace("\"", string.Empty).Replace("templateId:profile_v2_connectionsurl:", string.Empty));
}
catch { }
string PageSource = HttpHelper.getHtmlfromUrl1(new Uri(GroupUrl));
string[] array1 = Regex.Split(PageSource, "groupRegistration?");
List<string> ListGroupName = new List<string>();
string SelItem = string.Empty;
foreach (var itemGrps in array1)
{
try
{
if (itemGrps.Contains("?gid=") && !itemGrps.Contains("<!DOCTYPE html")) //">Join
{
if (itemGrps.IndexOf("?gid=") == 0)
{
try
{
int startindex = itemGrps.IndexOf("\"name\":");
string start = itemGrps.Substring(startindex);
int endIndex = start.IndexOf(",");
ListGroupName.Add(start.Substring(0, endIndex).Replace("\"", string.Empty).Replace("amp", string.Empty).Replace("&", string.Empty).Replace(";", string.Empty).Replace("csrfToken", string.Empty).Replace("name:", string.Empty));
}
catch { }
}
}
}
catch { }
}
foreach (var item in ListGroupName)
{
if (groupscollectin == string.Empty)
{
groupscollectin = item;
}
else
{
groupscollectin += " - " + item;
}
}
}
catch { }
if (string.IsNullOrEmpty(groupscollectin))
{
List<string> lstGroupData = new List<string>();
string[] array1 = Regex.Split(stringSource, "link_groups_settings\":?");
array1 = array1.Skip(1).ToArray();
foreach (string item in array1)
{
string _item = Utils.getBetween(item, "name\":", ",").Replace("name\":", "").Replace(":", "").Replace("\"", "");
lstGroupData.Add(_item);
}
foreach (var item in lstGroupData)
{
if (groupscollectin == string.Empty)
{
groupscollectin = item;
}
else
{
groupscollectin += " - " + item;
}
}
}
if (string.IsNullOrEmpty(groupscollectin))
{
List<string> lstGroupData = new List<string>();
string tempResponse = Utils.getBetween(stringSource, "<div id=\"groups\"", "<div>");
lstGroupData = HttpHelper.GetDataTag(tempResponse, "strong");
foreach (var item in lstGroupData)
{
if (groupscollectin == string.Empty)
{
groupscollectin = item;
}
else
{
groupscollectin += " - " + item;
}
}
}
#endregion
#region skill and Expertise
try
{
string[] strarr_skill = Regex.Split(stringSource, "endorse-item-name-text\"");
string[] strarr_skill1 = Regex.Split(stringSource, "fmt__skill_name\"");
if (strarr_skill.Count() >= 2)
{
foreach (string item in strarr_skill)
{
try
{
if (!item.Contains("!DOCTYPE html"))
{
try
{
//string Grp = item.Substring(item.IndexOf("<"), (item.IndexOf(">", item.IndexOf("<")) - item.IndexOf("<"))).Replace("<", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace("\"u002", "-").Trim();
string Grp = item.Substring(item.IndexOf(">"), (item.IndexOf("<", item.IndexOf(">")) - item.IndexOf(">"))).Replace(">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace("\"u002", "-").Trim();
checkgrplist.Add(Grp);
checkgrplist.Distinct().ToList();
}
catch { }
}
}
catch { }
}
foreach (string item in checkgrplist)
{
if (string.IsNullOrEmpty(Skill))
{
Skill = item.Replace("\"u002", "-").Trim();
}
else
{
Skill = Skill + " - " + item;
}
}
}
else
{
if (strarr_skill1.Count() >= 2)
{
try
{
foreach (string skillitem in strarr_skill1)
{
if (!skillitem.Contains("!DOCTYPE html"))
{
try
{
string Grp = skillitem.Substring(skillitem.IndexOf(":"), (skillitem.IndexOf("}", skillitem.IndexOf(":")) - skillitem.IndexOf(":"))).Replace(":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim();
checkgrplist.Add(Grp);
checkgrplist.Distinct().ToList();
}
catch { }
}
}
foreach (string item in checkgrplist)
{
if (string.IsNullOrEmpty(Skill))
{
Skill = item;
}
else
{
Skill = Skill + " - " + item;
}
}
}
catch (Exception ex)
{
}
}
}
}
catch (Exception ex)
{
}
Skill = Skill.Replace("a href=\";edu", "").Trim();
#endregion
#region Pasttitle
string[] pasttitles = Regex.Split(stringSource, "title_highlight");
string pstTitlesitem = string.Empty;
pasttitles = pasttitles.Skip(1).ToArray();
foreach (string item in pasttitles)
{
try
{
if (!item.Contains("<!DOCTYPE html>") && !item.Contains("Tip: You can also search by keyword"))
{
try
{
string[] Past_Ttl = Regex.Split(item, ",");
pstTitlesitem = Past_Ttl[0].Replace(":", string.Empty).Replace("\"", string.Empty).Replace("\\u002d", "-").Replace("&", "&");
}
catch { }
if ((pstTitlesitem.Contains("strong class")) || (pstTitlesitem.Contains("highlight")) || (string.IsNullOrEmpty(pstTitlesitem)))
{
try
{
int startindex1 = item.IndexOf("\"title\":");
string start1 = item.Substring(startindex1).Replace("\"title\":", string.Empty);
int endIndex1 = start1.IndexOf(",");
pstTitlesitem = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace(":", string.Empty).Replace("&", "&").Replace(";", string.Empty).Replace("\\u002d", "-").Replace("name:", string.Empty));
}
catch
{ }
}
if (string.IsNullOrEmpty(LDS_PastTitles))
{
LDS_PastTitles = pstTitlesitem;
}
else if (LDS_PastTitles.Contains(pstTitlesitem))
{
continue;
}
else
{
LDS_PastTitles = LDS_PastTitles + " : " + pstTitlesitem;
}
}
}
catch
{
}
}
#endregion
#region All Company Summary
//string[] pasttitles = Regex.Split(stringSource, "company_name");
//string pstTitlesitem = string.Empty;
string pstDescCompitem = string.Empty;
pasttitles = pasttitles.Skip(1).ToArray();
foreach (string item in pasttitles)
{
if (item.Contains("positionId"))
{
try
{
int startindex = item.IndexOf(":");
if (startindex > 0)
{
string start = item.Substring(startindex).Replace(":\"", "");
int endindex = start.IndexOf("\",");
string end = start.Substring(0, endindex);
pstTitlesitem = end.Replace(",", ";").Replace("&", "&").Replace("\\u002d", "-");
}
int startindex1 = item.IndexOf("summary_lb\":\"");
if (startindex > 0)
{
string start1 = item.Substring(startindex1).Replace("summary_lb\":\"", "");
int endindex1 = 0;
if (start1.Contains("associatedWith"))
{
endindex1 = start1.IndexOf("\",\"associatedWith\"");
if (start1.Contains("\"}"))
{
endindex1 = start1.IndexOf("\"}");
}
}
else if (start1.Contains("\"}"))
{
endindex1 = start1.IndexOf("\"}");
}
string end1 = start1.Substring(0, endindex1);
pstDescCompitem = end1.Replace(",", ";").Replace("u002d", "-").Replace("<br>", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("", "@").Replace("’", "'").Replace("•", "@").Replace("●", "@");
if (pstDescCompitem.Contains("\";\"associatedWith"))
{
pstDescCompitem = Regex.Split(pstDescCompitem, "\";\"associatedWith")[0];
}
}
if (string.IsNullOrEmpty(LDS_Desc_AllComp))
{
LDS_Desc_AllComp = pstTitlesitem + "-" + pstDescCompitem + " ++ ";
}
else
{
LDS_Desc_AllComp = LDS_Desc_AllComp + pstTitlesitem + "-" + pstDescCompitem + " ++ ";
}
}
catch
{
}
}
}
#endregion
#region AccountType
try
{
if (stringSource.Contains("has a Premium Account") || stringSource.Contains("Account Holder"))
{
AccountType = "Premium Account";
}
else
{
AccountType = "Basic Account";
}
}
catch (Exception ex)
{
}
#endregion Email
#region FullUrl
try
{
string[] UrlFull = System.Text.RegularExpressions.Regex.Split(Url, "&authType");
LDS_UserProfileLink = UrlFull[0];
LDS_UserProfileLink = Url;
// LDS_UserProfileLink = stringSource.Substring(stringSource.IndexOf("canonicalUrl\":"), (stringSource.IndexOf(",", stringSource.IndexOf("canonicalUrl\":")) - stringSource.IndexOf("canonicalUrl\":"))).Replace("canonicalUrl\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim();
}
catch { }
#endregion
LDS_LoginID = SearchCriteria.LoginID;
if (string.IsNullOrEmpty(firstname))
{
firstname = "Linkedin Member";
}
string endName = firstname + " " + lastname;
//GroupStatus.GroupSpecMem.Add(GroupMemId, endName);
if (firstname == string.Empty) firstname = "LinkedIn";
if (lastname == string.Empty) lastname = "Member";
if (Company == string.Empty) Company = "--";
if (titlecurrent == string.Empty) titlecurrent = "--";
if (companycurrent == string.Empty) companycurrent = "--";
if (LDS_Desc_AllComp == string.Empty) LDS_Desc_AllComp = "--";
if (LDS_BackGround_Summary == string.Empty) LDS_BackGround_Summary = "--";
if (Connection == string.Empty) Connection = "--";
if (recomandation == string.Empty) recomandation = "--";
if (Skill == string.Empty) Skill = "--";
if (LDS_Experience == string.Empty) LDS_Experience = "--";
if (EducationCollection == string.Empty) EducationCollection = "--";
if (groupscollectin == string.Empty) groupscollectin = "--";
if (USERemail == string.Empty) USERemail = "--";
if (LDS_UserContact == string.Empty) LDS_UserContact = "--";
if (LDS_PastTitles == string.Empty) LDS_PastTitles = "--";
if (AllComapny == string.Empty) AllComapny = "--";
if (location == string.Empty) location = "--";
if (country == string.Empty) country = "--";
if (Industry == string.Empty) Industry = "--";
if (Website == string.Empty) Website = "--";
string LDS_FinalData = TypeOfProfile + "," + LDS_UserProfileLink + "," + firstname + "," + lastname + "," + Company.Replace(",", ";") + "," + titlecurrent.Replace(",", ";") + "," + companycurrent.Replace(",", ";") + "," + CurrentCompSite.Replace(",", ";") + "," + LDS_BackGround_Summary.Replace(",", ";") + "," + Connection.Replace(",", ";") + "," + recomandation.Replace(",", string.Empty) + "," + Skill.Replace(",", ";") + "," + LDS_Experience.Replace(",", string.Empty) + "," + EducationCollection.Replace(",", ";") + "," + groupscollectin.Replace(",", ";") + "," + USERemail.Replace(",", ";") + "," + LDS_UserContact.Replace(",", ";") + "," + LDS_PastTitles + "," + AllComapny.Replace(",", ";") + "," + location.Replace(",", ";") + "," + country.Replace(",", ";") + "," + Industry.Replace(",", ";") + "," + Website.Replace(",", ";") + "," + LDS_LoginID + "," + AccountType;
if (!string.IsNullOrEmpty(firstname))
{
//Log("[ " + DateTime.Now + " ] => [ Data : " + LDS_FinalData + " ]");
}
else
{
AddToLogger("[ " + DateTime.Now + " ] => [ No Data For URL : " + Url + " ]");
GlobusFileHelper.AppendStringToTextfileNewLineWithCarat(Url, Globals.DesktopFolder + "\\UnScrapedList.txt");
}
if (SearchCriteria.starter)
{
//string tempFinalData = LDS_FinalData.Replace(";", "").Replace(LDS_UserProfileLink, "").Replace(TypeOfProfile, "").Replace(",", "").Replace(LDS_LoginID, "").Trim();
//if (!string.IsNullOrEmpty(tempFinalData))
{
//if (CheckEmployeeScraper)
//{
// string FileName = "CompanyEmployeeScraper";
// AppFileHelper.AddingLinkedInDataToCSVFileCompanyEmployeeScraper(LDS_FinalData, FileName);
// return true;
//}
//else if (CampaignScraper)
{
AppFileHelper.AddingLinkedInDataToCSVFile(LDS_FinalData, FileName);
return true;
}
//else
//{
// AppFileHelper.AddingLinkedInDataToCSVFile(LDS_FinalData, SearchCriteria.FileName);
// return true;
//}
}
}
}
catch { }
return false;
}
