public bool CrawlingLinkedInPage(string Url, ref GlobusHttpHelper HttpHelper,string FileName) { //Url = "http://www.linkedin.com/profile/view?id=44952194&authType=OUT_OF_NETWORK&authToken=DYBR&locale=en_US&srchid=3817933251416230963594&srchindex=2&srchtotal=1949&trk=vsrp_people_res_name&trkInfo=VSRPsearchId%3A3817933251416230963594%2CVSRPtargetId%3A44952194%2CVSRPcmpt%3Aprimary"; #region Data Initialization string GroupMemId = string.Empty; string Industry = string.Empty; string URLprofile = string.Empty; string firstname = string.Empty; string lastname = string.Empty; string location = string.Empty; string country = string.Empty; string postal = string.Empty; string phone = string.Empty; string USERemail = string.Empty; string code = string.Empty; string education1 = string.Empty; string education2 = string.Empty; string titlecurrent = string.Empty; string companycurrent = string.Empty; string CurrentCompUrl = string.Empty; string CurrentCompSite = string.Empty; string titlepast1 = string.Empty; string companypast1 = string.Empty; string titlepast2 = string.Empty; string html = string.Empty; string companypast2 = string.Empty; string titlepast3 = string.Empty; string companypast3 = string.Empty; string titlepast4 = string.Empty; string companypast4 = string.Empty; string Recommendations = string.Empty; string Connection = string.Empty; string Designation = string.Empty; string Website = string.Empty; string Contactsettings = string.Empty; string recomandation = string.Empty; string titleCurrenttitle = string.Empty; string titleCurrenttitle2 = string.Empty; string titleCurrenttitle3 = string.Empty; string titleCurrenttitle4 = string.Empty; string Skill = string.Empty; string TypeOfProfile = "Public"; List<string> EducationList = new List<string>(); string Finaldata = string.Empty; string EducationCollection = string.Empty; List<string> checkerlst = new List<string>(); List<string> checkgrplist = new List<string>(); string groupscollectin = string.Empty; string strFamilyName = string.Empty; string LDS_LoginID = string.Empty; string LDS_Websites = string.Empty; string LDS_UserProfileLink = string.Empty; string LDS_CurrentTitle = string.Empty; string LDS_Experience = string.Empty; string LDS_UserContact = string.Empty; string LDS_PastTitles = string.Empty; string LDS_BackGround_Summary = string.Empty; string LDS_Desc_AllComp = string.Empty; string Company = string.Empty; List<string> lstpasttitle = new List<string>(); List<string> checkpasttitle = new List<string>(); string DeegreeConn = string.Empty; string AccountType = string.Empty; bool CheckEmployeeScraper = false; string fileName = string.Empty; bool CampaignScraper = false; #endregion #region GetRequest if (Url.Contains("CompanyEmployeeScraper")) { try { Url = Url.Replace("CompanyEmployeeScraper", string.Empty); CheckEmployeeScraper = true; } catch { } } if (Url.Contains("CampaignScraper")) { try { string[] Url_Split = Regex.Split(Url, "CampaignScraper"); Url = Url_Split[0]; fileName = Url_Split[1]; CampaignScraper = true; } catch { } } string stringSource = HttpHelper.getHtmlfromUrl(new Uri(Url.Replace("http", "https"))); #endregion #region GroupMemId try { string[] gid = Url.Split('&'); GroupMemId = gid[0].Replace("http://www.linkedin.com/profile/view?id=", string.Empty); } catch { } #endregion #region Name try { try { strFamilyName = stringSource.Substring(stringSource.IndexOf("\"See Full Name\",\"i18n_Edit\":\"Edit\",\"fmt__full_name\":\""), (stringSource.IndexOf("i18n__expand_your_network_to_see_more", stringSource.IndexOf("\"See Full Name\",\"i18n_Edit\":\"Edit\",\"fmt__full_name\":\"")) - stringSource.IndexOf("\"See Full Name\",\"i18n_Edit\":\"Edit\",\"fmt__full_name\":\""))).Replace("\"See Full Name\",\"i18n_Edit\":\"Edit\",\"fmt__full_name\":\"", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim(); } catch { try { strFamilyName = stringSource.Substring(stringSource.IndexOf("fmt__full_name\":"), (stringSource.IndexOf(",", stringSource.IndexOf("fmt__full_name\":")) - stringSource.IndexOf("fmt__full_name\":"))).Replace("fmt__full_name\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim(); } catch { } } if (string.IsNullOrEmpty(strFamilyName)) { try { strFamilyName = stringSource.Substring(stringSource.IndexOf("<span class=\"full-name\">"), (stringSource.IndexOf("</span><span></span></span></h1></div></div><div id=\"headline-container\" data-li-template=\"headline\">", stringSource.IndexOf("</span><span></span></span></h1></div></div><div id=\"headline-container\" data-li-template=\"headline\">")) - stringSource.IndexOf("<span class=\"full-name\">"))).Replace("<span class=\"full-name\">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim(); } catch { } } if (string.IsNullOrEmpty(strFamilyName)) { try { int StartIndex = stringSource.IndexOf("profileFullName"); string Start = stringSource.Substring(StartIndex).Replace("profileFullName", string.Empty); int EndIndex = Start.IndexOf("slideData"); string End = Start.Substring(0, EndIndex).Replace(":", string.Empty).Replace("'",string.Empty).Replace(",",string.Empty).Trim(); strFamilyName = End.Trim(); } catch { } } } catch { } if (string.IsNullOrEmpty(strFamilyName)) { try { int StartIndex = stringSource.IndexOf("</script><title>"); string Start = stringSource.Substring(StartIndex).Replace("</script><title>", string.Empty); int EndIndex = Start.IndexOf("| LinkedIn</title>"); string End = Start.Substring(0, EndIndex).Replace(":", string.Empty).Replace("'", string.Empty).Replace(",", string.Empty).Trim(); strFamilyName = End.Trim(); } catch { } } // #endregion #region Namesplitation string[] NameArr = new string[5]; if (strFamilyName.Contains(" ")) { try { NameArr = Regex.Split(strFamilyName, " "); } catch { } } #endregion #region FirstName try { firstname = NameArr[0]; } catch { } #endregion #region LastName try { lastname = NameArr[1]; } catch { } try { if (NameArr.Count() == 3) { try { lastname = NameArr[1] + " " + NameArr[2]; } catch { } } if (lastname.Contains("}]")) { #region Name try { try { strFamilyName = stringSource.Substring(stringSource.IndexOf("<span class=\"n fn\">")).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim(); } catch { try { strFamilyName = stringSource.Substring(stringSource.IndexOf("fmt__full_name\":"), (stringSource.IndexOf(",", stringSource.IndexOf("fmt__full_name\":")) - stringSource.IndexOf("fmt__full_name\":"))).Replace("fmt__full_name\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Trim(); } catch { } } } catch { } #endregion } } catch { } #endregion #region Company try { try { try { //soft engg at TCSi18n__LocationLocationi18n__Linkedin_memberLinkedIn Member //Company = stringSource.Substring(stringSource.IndexOf("\"memberHeadline"), (stringSource.IndexOf("memberID", stringSource.IndexOf("\"memberHeadline")) - stringSource.IndexOf("\"memberHeadline"))).Replace("\"memberHeadline", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace(" ", string.Empty).Replace("visibletrue", string.Empty).Replace("isLNLedtrue",string.Empty).Replace("isPortfoliofalse",string.Empty).Replace("i18n__Location",string.Empty).Replace("Locationi18n__Linkedin_member",string.Empty).Replace("u002d","-").Replace("LinkedIn Member",string.Empty).Replace("--Location","--").Trim(); Company = stringSource.Substring(stringSource.IndexOf("\"memberHeadline"), (stringSource.IndexOf("memberID", stringSource.IndexOf("\"memberHeadline")) - stringSource.IndexOf("\"memberHeadline"))).Replace("\"memberHeadline", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace(" ", string.Empty).Replace("visibletrue", string.Empty).Replace("isLNLedtrue", string.Empty).Replace("isPortfoliofalse", string.Empty).Replace("i18n__Location", string.Empty).Replace("Locationi18n__Linkedin_member", string.Empty).Replace("u002d", "-").Replace("LinkedIn Member", string.Empty).Replace("--Location", "--").ToString().Trim(); if (Company.Contains("#Name?")) { Company = "--"; } if (Company.Contains("i18n")) { Company = Regex.Split(Company, "i18n")[0]; } } catch { } if (string.IsNullOrEmpty(Company)) { try { Company = stringSource.Substring(stringSource.IndexOf("memberHeadline\":"), (stringSource.IndexOf(",", stringSource.IndexOf("memberHeadline\":")) - stringSource.IndexOf("memberHeadline\":"))).Replace("memberHeadline\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace(" ", string.Empty).Replace(":", "").Replace("visibletrue", string.Empty).Replace("&dsh;", "").Replace("&", "").Replace(";", "").Replace("isLNLedtrue", string.Empty).Replace("isPortfoliofalse", string.Empty).Trim(); } catch { } } if (string.IsNullOrEmpty(Company)) { try { Company = stringSource.Substring(stringSource.IndexOf("class=\"title \">"), (stringSource.IndexOf("</p></div></div><div class=\"demographic-info adr editable-item\" id=\"demographics\">", stringSource.IndexOf("</p></div></div><div class=\"demographic-info adr editable-item\" id=\"demographics\">")) - stringSource.IndexOf("class=\"title \">"))).Replace("class=\"title \">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace("<strong class=highlight>", string.Empty).Replace("</strong>", string.Empty).Trim(); } catch { } } string[] strdesigandcompany = new string[4]; if (Company.Contains(" at ")) { try { strdesigandcompany = Regex.Split(Company, " at "); } catch { } #region Title try { titlecurrent = strdesigandcompany[0]; } catch { } #endregion #region Current Company try { companycurrent = strdesigandcompany[1]; } catch { } #endregion } } catch { } #region Current Company Site try { try { CurrentCompUrl = stringSource.Substring(stringSource.IndexOf("/companies"), (stringSource.IndexOf("/companies", stringSource.IndexOf("?miniprofile")) - stringSource.IndexOf("?miniprofile"))).Replace("?miniprofi", string.Empty).ToString().Trim(); CurrentCompUrl = "https://www.linkedin.com" + CurrentCompUrl; } catch { } string CompanyUrl = HttpHelper.getHtmlfromUrl1(new Uri(CurrentCompUrl)); try { CurrentCompSite = CompanyUrl.Substring(CompanyUrl.IndexOf("<dt>Website</dt>"), (CompanyUrl.IndexOf("</dd>", CompanyUrl.IndexOf("<dt>Website</dt>")) - CompanyUrl.IndexOf("<dt>Website</dt>"))).Replace("<dt>Website</dt>", string.Empty).Replace("<dd>", string.Empty).Trim(); } catch { } try { CurrentCompSite = CompanyUrl.Substring(CompanyUrl.IndexOf("<h4>Website</h4>"), (CompanyUrl.IndexOf("</p>", CompanyUrl.IndexOf("<h4>Website</h4>")) - CompanyUrl.IndexOf("<h4>Website</h4>"))).Replace("<h4>Website</h4>", string.Empty).Replace("<p>", string.Empty).Trim(); if (CurrentCompSite.Contains("a href=")) { try { string[] websArr = Regex.Split(CurrentCompSite, ">"); CurrentCompSite = websArr[1].Replace("</a", string.Empty).Replace("\n", string.Empty).Trim(); ; } catch { } } } catch { } } catch { } #endregion #region PastCompany string[] companylist = Regex.Split(stringSource, "companyName\""); if (companylist.Count() == 1) { companylist = Regex.Split(stringSource, "company-name"); } if (companylist.Count() == 1) { //companylist = Regex.Split(stringSource, "Companies"); } string AllComapny = string.Empty; string Companyname = string.Empty; if (!stringSource.Contains("company-name") && companylist.Count() > 1) { foreach (string item in companylist) { try { if (!item.Contains("<!DOCTYPE html>")) { Companyname = item.Substring(item.IndexOf(":"), (item.IndexOf(",", item.IndexOf(":")) - item.IndexOf(":"))).Replace(":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace("}", string.Empty).Replace("]", string.Empty).Trim(); string items = item; checkerlst.Add(Companyname); checkerlst = checkerlst.Distinct().ToList(); } } catch { } } } else { foreach (string item in companylist) { try { if (!item.Contains("<!DOCTYPE html>")) { Companyname = item.Substring(item.IndexOf(">"), (item.IndexOf("<", item.IndexOf(">")) - item.IndexOf(">"))).Replace(">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace("}", string.Empty).Replace("]", string.Empty).Trim(); string items = item; if (!string.IsNullOrEmpty(Companyname)) { checkerlst.Add(Companyname); checkerlst = checkerlst.Distinct().ToList(); } } } catch { } } } AllComapny = string.Empty; foreach (string item1 in checkerlst) { if (string.IsNullOrEmpty(AllComapny)) { AllComapny = item1.Replace("}", "").Replace("]", "").Replace("&", "&"); } else { AllComapny = AllComapny + " : " + item1.Replace("}", "").Replace("]", "").Replace("&", "&"); } } #endregion #endregion Company #region Company Descripription try { string[] str_CompanyDesc = Regex.Split(stringSource, "showSummarySection"); foreach (string item in str_CompanyDesc) { try { string Current_Company = string.Empty; if (!item.Contains("<!DOCTYPE html>")) { int startindex = item.IndexOf("specialties\":\""); if (startindex > 0) { try { string start = item.Substring(startindex).Replace("specialties\":", ""); int endindex = start.IndexOf("\",\"associatedWith\""); string end = start.Substring(0, endindex); Current_Company = end.Replace(",\"specialties_lb\":", string.Empty).Replace("\"", string.Empty).Replace("summary_lb", "Summary").Replace("•", ";").Replace("<br>", string.Empty).Replace("\\n", string.Empty).Replace("\"u002", "-"); LDS_BackGround_Summary = Current_Company; } catch { } } } if (!item.Contains("<!DOCTYPE html>")) { int startindex = item.IndexOf("\"summary_lb\""); if (startindex > 0) { try { string start = item.Substring(startindex).Replace("\"summary_lb\"", ""); int endindex = start.IndexOf("\",\"associatedWith\""); string end = start.Substring(0, endindex); Current_Company = end.Replace(",\"specialties_lb\":", string.Empty).Replace("<br>", string.Empty).Replace("\n", string.Empty).Replace("\"", string.Empty).Replace("summary_lb", "Summary").Replace(",", ";").Replace("u002", "-").Replace("•", string.Empty).Replace(":", string.Empty); LDS_BackGround_Summary = Current_Company; } catch { } } } } catch { } } if (string.IsNullOrEmpty(LDS_BackGround_Summary)) { try { LDS_BackGround_Summary = HttpHelper.GetDataWithTagValueByTagAndAttributeNameWithId(stringSource, "div", "summary-item-view"); LDS_BackGround_Summary = Regex.Replace(LDS_BackGround_Summary, "<.*?>", string.Empty).Replace(",", "").Replace("\n", "").Replace("<![CDATA[", "").Trim(); } catch { } } } catch { } #endregion #region Education try { string[] str_UniversityName = Regex.Split(stringSource, "link__school_name"); foreach (string item in str_UniversityName) { try { string School = string.Empty; string Degree = string.Empty; string SessionEnd = string.Empty; string SessionStart = string.Empty; string Education = string.Empty; if (stringSource.Contains("link__school_name")) { if (!item.Contains("<!DOCTYPE html>")) { try { try { int startindex = item.IndexOf("fmt__school_highlight"); string start = item.Substring(startindex).Replace("fmt__school_highlight", ""); int endindex = start.IndexOf(","); School = start.Substring(0, endindex).Replace("\\u002d", string.Empty).Replace(":", string.Empty).Replace("\"", string.Empty).Replace("_highlight", string.Empty); } catch { } try { int startindex1 = item.IndexOf("degree"); string start1 = item.Substring(startindex1).Replace("degree", ""); int endindex1 = start1.IndexOf(","); Degree = start1.Substring(0, endindex1).Replace("\\u002d", string.Empty).Replace(":", string.Empty).Replace("\"", string.Empty).Replace("_highlight", string.Empty); } catch { } try { int startindex2 = item.IndexOf("enddate_my"); string start2 = item.Substring(startindex2).Replace("enddate_my", ""); int endindex2 = start2.IndexOf(","); SessionEnd = start2.Substring(0, endindex2).Replace("\\u002d", string.Empty).Replace(":", string.Empty).Replace("\"", string.Empty).Replace("_highlight", string.Empty); } catch { } try { int startindex3 = item.IndexOf("startdate_my"); string start3 = item.Substring(startindex3).Replace("startdate_my", ""); int endindex3 = start3.IndexOf(","); SessionStart = start3.Substring(0, endindex3).Replace("\\u002d", string.Empty).Replace(":", string.Empty).Replace("\"", string.Empty); } catch { } if (SessionStart == string.Empty && SessionEnd == string.Empty) { Education = " [" + School + "] Degree: " + Degree; } else { Education = " [" + School + "] Degree: " + Degree + " Session: " + SessionStart + "-" + SessionEnd; } //University = item.Substring(item.IndexOf(":"), (item.IndexOf(",", item.IndexOf(":")) - item.IndexOf(":"))).Replace(":", string.Empty).Replace("\\u002d", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); } catch { } EducationList.Add(Education); } } else { str_UniversityName = Regex.Split(stringSource, "<div class=\"education"); foreach (string tempItem in str_UniversityName) { try { if (!tempItem.Contains("<!DOCTYPE html>")) { List<string> lstSchool = HttpHelper.GetTextDataByTagAndAttributeName(tempItem, "h4", "summary fn org"); List<string> lstDegree = HttpHelper.GetTextDataByTagAndAttributeName(tempItem, "span", "degree"); List<string> lstSession = HttpHelper.GetTextDataByTagAndAttributeName(tempItem, "span", "education-date"); if (lstSession.Count == 0) { Education = " [" + lstSchool[0] + "] Degree: " + lstDegree[0]; } else { Education = " [" + lstSchool[0] + "] Degree: " + lstDegree[0] + " Session: " + lstSession[0].Replace("–", "-").Replace(",", "").Trim(); } EducationList.Add(Education); } } catch { } } } } catch { } } EducationList = EducationList.Distinct().ToList(); foreach (string item in EducationList) { if (string.IsNullOrEmpty(EducationCollection)) { EducationCollection = item.Replace("}", "").Replace("]", "").Replace("&", "&"); } else { EducationCollection = EducationCollection + " - " + item.Replace("}", "").Replace("]", "").Replace("&", "&"); } } // string University1 = stringSource.Substring(stringSource.IndexOf("schoolName\":"), (stringSource.IndexOf(",", stringSource.IndexOf("schoolName\":")) - stringSource.IndexOf("schoolName\":"))).Replace("schoolName\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); } catch { } #endregion Education #region Email try { if (stringSource.Contains("mailto:")) { string[] str_Email = Regex.Split(stringSource, "mailto:"); USERemail = stringSource.Substring(stringSource.IndexOf("mailto:"), (stringSource.IndexOf(">", stringSource.IndexOf("mailto:")) - stringSource.IndexOf("mailto:"))).Replace("mailto:", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); } else { string[] str_Email = Regex.Split(stringSource, "email\""); USERemail = stringSource.Substring(stringSource.IndexOf("[{\"email\":"), (stringSource.IndexOf("}]", stringSource.IndexOf("[{\"email\":")) - stringSource.IndexOf("[{\"email\":"))).Replace("[{\"email\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); } } catch (Exception ex) { } #endregion Email #region UserContact try { if (stringSource.Contains("<div id=\"phone-view\">")) { //string[] str_Contact = Regex.Split(stringSource, "<div id=\"phone-view\">"); LDS_UserContact = stringSource.Substring(stringSource.IndexOf("<div id=\"phone-view\">"), (stringSource.IndexOf("</li>", stringSource.IndexOf("<div id=\"phone-view\">")) - stringSource.IndexOf("<div id=\"phone-view\">"))).Replace("<div id=\"phone-view\">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace("<ul><li>", string.Empty).Replace(" ", "").Trim(); } else { //string[] str_Email = Regex.Split(stringSource, "<div id=\"phone-view\">"); LDS_UserContact = stringSource.Substring(stringSource.IndexOf("<div id=\"phone-view\">"), (stringSource.IndexOf("</li>", stringSource.IndexOf("<div id=\"phone-view\">")) - stringSource.IndexOf("<div id=\"phone-view\">"))).Replace("<div id=\"phone-view\">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace("<ul><li>", string.Empty).Replace(" ", "").Trim(); } } catch (Exception ex) { } #endregion Email #region Website try { Website = stringSource.Substring(stringSource.IndexOf("[{\"URL\":"), (stringSource.IndexOf(",", stringSource.IndexOf("[{\"URL\":")) - stringSource.IndexOf("[{\"URL\":"))).Replace("[{\"URL\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace("}", string.Empty).Replace("]", string.Empty).Trim(); } catch { } if (string.IsNullOrEmpty(Website)) { try { Website = HttpHelper.GetDataWithTagValueByTagAndAttributeNameWithId(stringSource, "div", "website-view"); Website = Regex.Replace(Website, "<[^>]*>", String.Empty).Replace("\n", "").Replace("\r", "").Replace(" ", " ").Trim(); Website = Regex.Replace(Website, @"\s+", " ").Replace(",", " ").Trim(); } catch { } } #endregion Website #region location try { //location = stringSource.Substring(stringSource.IndexOf("Country\",\"fmt__location\":"), (stringSource.IndexOf("i18n_no_location_matches", stringSource.IndexOf("Country\",\"fmt__location\":")) - stringSource.IndexOf("Country\",\"fmt__location\":"))).Replace("Country\",\"fmt__location\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); int startindex = stringSource.IndexOf("fmt_location"); string start = stringSource.Substring(startindex).Replace("fmt_location\":\"", ""); int endindex = start.IndexOf("\""); string end = start.Substring(0, endindex).Replace("\u002d", string.Empty); location = end; } catch (Exception ex) { } if (string.IsNullOrEmpty(location)) { try { List<string> lstLocation = HttpHelper.GetTextDataByTagAndAttributeName(stringSource, "span", "locality"); if (lstLocation.Count > 0) { location = lstLocation[lstLocation.Count - 1].Trim(); } } catch { } } #endregion location #region Country try { int startindex = stringSource.IndexOf("\"geo_region\":"); if (startindex > 0) { string start = stringSource.Substring(startindex).Replace("\"geo_region\":", ""); int endindex = start.IndexOf("\"i18n_geo_region\":\"Location\""); string end = start.Substring(0, endindex); country = end; string[] array = Regex.Split(end, "\"name\":\""); array = array.Skip(1).ToArray(); foreach (string item in array) { try { int startindex1 = item.IndexOf("\",\""); string strat1 = item.Substring(0, startindex1); country = strat1; break; } catch (Exception ex) { } } } } catch (Exception ex) { } if (country == string.Empty) { try { string[] countLocation = location.Split(','); if (countLocation.Count() == 2) { country = location.Split(',')[1]; } else if (countLocation.Count() == 3) { country = location.Split(',')[2]; } } catch { } } #endregion #region Industry try { //Industry = stringSource.Substring(stringSource.IndexOf("fmt__industry_highlight\":"), (stringSource.IndexOf(",", stringSource.IndexOf("fmt__industry_highlight\":")) - stringSource.IndexOf("fmt__industry_highlight\":"))).Replace("fmt__industry_highlight\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); int startindex = stringSource.IndexOf("\"industry_highlight\":\""); if (startindex > 0) { string start = stringSource.Substring(startindex).Replace("\"industry_highlight\":\"", ""); int endindex = start.IndexOf("\","); string end = start.Substring(0, endindex).Replace("\"", string.Empty).Replace("</strong>", string.Empty).Replace("&", "&"); if (end.Contains("strong class")) { Industry = end.Split('>')[1]; } else { Industry = end; } } } catch (Exception ex) { } if (string.IsNullOrEmpty(Industry)) { try { List<string> lstIndustry = HttpHelper.GetTextDataByTagAndAttributeName(stringSource, "dd", "industry"); if (lstIndustry.Count > 0) { Industry = lstIndustry[0].Replace(",", ":").Trim(); } } catch { } } #endregion Industry #region Connection try { //Connection = stringSource.Substring(stringSource.IndexOf("_count_string\":"), (stringSource.IndexOf(",", stringSource.IndexOf("_count_string\":")) - stringSource.IndexOf("_count_string\":"))).Replace("_count_string\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); int startindex = stringSource.IndexOf("\"numberOfConnections\":"); if (startindex > 0) { string start = stringSource.Substring(startindex).Replace("\"numberOfConnections\":", ""); int endindex = start.IndexOf(","); string end = start.Substring(0, endindex).Replace("&", "&").Replace("}", string.Empty); Connection = end; } } catch (Exception ex) { } if (string.IsNullOrEmpty(Connection)) { try { List<string> lstConnection = HttpHelper.GetTextDataByTagAndAttributeName(stringSource, "div", "member-connections"); if (lstConnection.Count > 0) { Connection = lstConnection[0].Replace(",", ":").Trim(); } } catch { } } #endregion Connection #region Recommendation try { string RecomnedUrl = string.Empty; try { int startindex = stringSource.IndexOf("endorsements?id="); string start = stringSource.Substring(startindex); int endIndex = start.IndexOf("\"mem_pic\":"); if (endIndex < 0) { endIndex = start.IndexOf(">"); } RecomnedUrl = (start.Substring(0, endIndex).Replace(",", string.Empty).Replace("\"", string.Empty).Replace(":", string.Empty)); } catch { } string PageSource = HttpHelper.getHtmlfromUrl1(new Uri("http://www.linkedin.com/profile/profile-v2-" + RecomnedUrl + "")); string[] arrayRecommendedName = Regex.Split(PageSource, "headline"); if (arrayRecommendedName.Count() == 1) { arrayRecommendedName = Regex.Split(PageSource, "fmt__recommendeeFullName"); } List<string> ListRecommendationName = new List<string>(); foreach (var itemRecomName in arrayRecommendedName) { try { if (!itemRecomName.Contains("Endorsements")) { string Heading = string.Empty; string Name = string.Empty; try { int startindex = itemRecomName.IndexOf(":"); string start = itemRecomName.Substring(startindex); int endIndex = start.IndexOf("\","); Heading = (start.Substring(0, endIndex).Replace("\"", string.Empty).Replace(":", string.Empty).Replace(",", ";")); } catch { } try { int startindex1 = itemRecomName.IndexOf("fmt__referrerfullName"); string start1 = itemRecomName.Substring(startindex1); int endIndex1 = start1.IndexOf("\","); Name = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace("fmt__referrerfullName", string.Empty).Replace(":", string.Empty)); } catch { } if (Name == string.Empty) { int startindex1 = itemRecomName.IndexOf("recommenderTitle\":"); string start1 = itemRecomName.Substring(startindex1); int endIndex1 = start1.IndexOf("\","); Name = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace("recommenderTitle", string.Empty).Replace(":", string.Empty).Replace(",", string.Empty)); } ListRecommendationName.Add(Name + " : " + Heading); } } catch { } } foreach (var item in ListRecommendationName) { if (recomandation == string.Empty) { recomandation = item; } else { recomandation += " - " + item; } } } catch { } #endregion #region Following #endregion #region Experience if (LDS_Experience == string.Empty) { try { string[] array = Regex.Split(stringSource, "title_highlight"); string exp = string.Empty; string comp = string.Empty; List<string> ListExperince = new List<string>(); string SelItem = string.Empty; if (stringSource.Contains("title_highlight")) { foreach (var itemGrps in array) { try { if (itemGrps.Contains("title_pivot") && !itemGrps.Contains("<!DOCTYPE html")) //">Join { try { int startindex = itemGrps.IndexOf("\":\""); string start = itemGrps.Substring(startindex); int endIndex = start.IndexOf(","); exp = (start.Substring(0, endIndex).Replace("\"", string.Empty).Replace(":", string.Empty).Replace("&", "&").Replace(";", string.Empty).Replace("\\u002d", "-").Replace("name:", string.Empty)); } catch { } if ((exp.Contains("strong class")) || (exp.Contains("highlight")) || (string.IsNullOrEmpty(exp))) { try { int startindex1 = itemGrps.IndexOf("\"title\":"); string start1 = itemGrps.Substring(startindex1).Replace("\"title\":", string.Empty); int endIndex1 = start1.IndexOf(","); exp = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace(":", string.Empty).Replace("&", "&").Replace(";", string.Empty).Replace("\\u002d", "-").Replace("name:", string.Empty)); } catch { } } try { int startindex1 = itemGrps.IndexOf("companyName"); string start1 = itemGrps.Substring(startindex1); int endIndex1 = start1.IndexOf(","); comp = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace("companyName", string.Empty).Replace(":", string.Empty).Replace(";", string.Empty).Replace("\\u002d", "-").Replace("name:", string.Empty)); } catch { } if (titlecurrent == string.Empty) { titlecurrent = exp; } if (companycurrent == string.Empty) { companycurrent = comp; } ListExperince.Add(exp + ":" + comp); } } catch { } } } else { array = Regex.Split(stringSource, "<header>"); foreach (string tempItem in array) { try { if (!tempItem.Contains("<!DOCTYPE html>")) { List<string> lstExp = objChilkat.GetDataTag(tempItem, "h4"); List<string> lstComp = objChilkat.GetDataTag(tempItem, "h5"); if (lstExp.Count > 0) { exp = lstExp[0]; } if (lstComp.Count > 0) { comp = lstComp[0]; if (string.IsNullOrEmpty(comp)) { comp = lstComp[1]; } } if (titlecurrent == string.Empty) { titlecurrent = lstExp[0]; } if (companycurrent == string.Empty) { companycurrent = lstComp[0]; } ListExperince.Add(exp + ":" + comp); } } catch { } } } foreach (var item in ListExperince) { if (LDS_Experience == string.Empty) { LDS_Experience = item; } else { LDS_Experience += " - " + item; } } } catch { } try { if (string.IsNullOrEmpty(titlecurrent)) { int StartIndex = stringSource.IndexOf("trk=prof-0-ovw-curr_pos\">"); string Start = stringSource.Substring(StartIndex).Replace("trk=prof-0-ovw-curr_pos\">", string.Empty); int EndIndex = Start.IndexOf("</a>"); string End = Start.Substring(0, EndIndex).Replace("</a>", string.Empty); titlecurrent = End.Trim(); } } catch { } } #endregion #region Group try { string GroupUrl = string.Empty; try { int startindex = stringSource.IndexOf("templateId\":\"profile_v2_connections"); string start = stringSource.Substring(startindex); //int endIndex = start.IndexOf("vsrp_people_res_name"); int endIndex = start.IndexOf("}"); GroupUrl = (start.Substring(0, endIndex).Replace(",", string.Empty).Replace("\"", string.Empty).Replace("templateId:profile_v2_connectionsurl:", string.Empty)); } catch { } string PageSource = HttpHelper.getHtmlfromUrl1(new Uri(GroupUrl)); string[] array1 = Regex.Split(PageSource, "groupRegistration?"); List<string> ListGroupName = new List<string>(); string SelItem = string.Empty; foreach (var itemGrps in array1) { try { if (itemGrps.Contains("?gid=") && !itemGrps.Contains("<!DOCTYPE html")) //">Join { if (itemGrps.IndexOf("?gid=") == 0) { try { int startindex = itemGrps.IndexOf("\"name\":"); string start = itemGrps.Substring(startindex); int endIndex = start.IndexOf(","); ListGroupName.Add(start.Substring(0, endIndex).Replace("\"", string.Empty).Replace("amp", string.Empty).Replace("&", string.Empty).Replace(";", string.Empty).Replace("csrfToken", string.Empty).Replace("name:", string.Empty)); } catch { } } } } catch { } } foreach (var item in ListGroupName) { if (groupscollectin == string.Empty) { groupscollectin = item; } else { groupscollectin += " - " + item; } } } catch { } if (string.IsNullOrEmpty(groupscollectin)) { List<string> lstGroupData = new List<string>(); string[] array1 = Regex.Split(stringSource, "link_groups_settings\":?"); array1 = array1.Skip(1).ToArray(); foreach (string item in array1) { string _item = Utils.getBetween(item, "name\":", ",").Replace("name\":", "").Replace(":", "").Replace("\"", ""); lstGroupData.Add(_item); } foreach (var item in lstGroupData) { if (groupscollectin == string.Empty) { groupscollectin = item; } else { groupscollectin += " - " + item; } } } if (string.IsNullOrEmpty(groupscollectin)) { List<string> lstGroupData = new List<string>(); string tempResponse = Utils.getBetween(stringSource, "<div id=\"groups\"", "<div>"); lstGroupData = HttpHelper.GetDataTag(tempResponse, "strong"); foreach (var item in lstGroupData) { if (groupscollectin == string.Empty) { groupscollectin = item; } else { groupscollectin += " - " + item; } } } #endregion #region skill and Expertise try { string[] strarr_skill = Regex.Split(stringSource, "endorse-item-name-text\""); string[] strarr_skill1 = Regex.Split(stringSource, "fmt__skill_name\""); if (strarr_skill.Count() >= 2) { foreach (string item in strarr_skill) { try { if (!item.Contains("!DOCTYPE html")) { try { //string Grp = item.Substring(item.IndexOf("<"), (item.IndexOf(">", item.IndexOf("<")) - item.IndexOf("<"))).Replace("<", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace("\"u002", "-").Trim(); string Grp = item.Substring(item.IndexOf(">"), (item.IndexOf("<", item.IndexOf(">")) - item.IndexOf(">"))).Replace(">", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Replace("\"u002", "-").Trim(); checkgrplist.Add(Grp); checkgrplist.Distinct().ToList(); } catch { } } } catch { } } foreach (string item in checkgrplist) { if (string.IsNullOrEmpty(Skill)) { Skill = item.Replace("\"u002", "-").Trim(); } else { Skill = Skill + " - " + item; } } } else { if (strarr_skill1.Count() >= 2) { try { foreach (string skillitem in strarr_skill1) { if (!skillitem.Contains("!DOCTYPE html")) { try { string Grp = skillitem.Substring(skillitem.IndexOf(":"), (skillitem.IndexOf("}", skillitem.IndexOf(":")) - skillitem.IndexOf(":"))).Replace(":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); checkgrplist.Add(Grp); checkgrplist.Distinct().ToList(); } catch { } } } foreach (string item in checkgrplist) { if (string.IsNullOrEmpty(Skill)) { Skill = item; } else { Skill = Skill + " - " + item; } } } catch (Exception ex) { } } } } catch (Exception ex) { } Skill = Skill.Replace("a href=\";edu", "").Trim(); #endregion #region Pasttitle string[] pasttitles = Regex.Split(stringSource, "title_highlight"); string pstTitlesitem = string.Empty; pasttitles = pasttitles.Skip(1).ToArray(); foreach (string item in pasttitles) { try { if (!item.Contains("<!DOCTYPE html>") && !item.Contains("Tip: You can also search by keyword")) { try { string[] Past_Ttl = Regex.Split(item, ","); pstTitlesitem = Past_Ttl[0].Replace(":", string.Empty).Replace("\"", string.Empty).Replace("\\u002d", "-").Replace("&", "&"); } catch { } if ((pstTitlesitem.Contains("strong class")) || (pstTitlesitem.Contains("highlight")) || (string.IsNullOrEmpty(pstTitlesitem))) { try { int startindex1 = item.IndexOf("\"title\":"); string start1 = item.Substring(startindex1).Replace("\"title\":", string.Empty); int endIndex1 = start1.IndexOf(","); pstTitlesitem = (start1.Substring(0, endIndex1).Replace("\"", string.Empty).Replace(":", string.Empty).Replace("&", "&").Replace(";", string.Empty).Replace("\\u002d", "-").Replace("name:", string.Empty)); } catch { } } if (string.IsNullOrEmpty(LDS_PastTitles)) { LDS_PastTitles = pstTitlesitem; } else if (LDS_PastTitles.Contains(pstTitlesitem)) { continue; } else { LDS_PastTitles = LDS_PastTitles + " : " + pstTitlesitem; } } } catch { } } #endregion #region All Company Summary //string[] pasttitles = Regex.Split(stringSource, "company_name"); //string pstTitlesitem = string.Empty; string pstDescCompitem = string.Empty; pasttitles = pasttitles.Skip(1).ToArray(); foreach (string item in pasttitles) { if (item.Contains("positionId")) { try { int startindex = item.IndexOf(":"); if (startindex > 0) { string start = item.Substring(startindex).Replace(":\"", ""); int endindex = start.IndexOf("\","); string end = start.Substring(0, endindex); pstTitlesitem = end.Replace(",", ";").Replace("&", "&").Replace("\\u002d", "-"); } int startindex1 = item.IndexOf("summary_lb\":\""); if (startindex > 0) { string start1 = item.Substring(startindex1).Replace("summary_lb\":\"", ""); int endindex1 = 0; if (start1.Contains("associatedWith")) { endindex1 = start1.IndexOf("\",\"associatedWith\""); if (start1.Contains("\"}")) { endindex1 = start1.IndexOf("\"}"); } } else if (start1.Contains("\"}")) { endindex1 = start1.IndexOf("\"}"); } string end1 = start1.Substring(0, endindex1); pstDescCompitem = end1.Replace(",", ";").Replace("u002d", "-").Replace("<br>", string.Empty).Replace("\\n", string.Empty).Replace("\\", string.Empty).Replace("", "@").Replace("’", "'").Replace("•", "@").Replace("●", "@"); if (pstDescCompitem.Contains("\";\"associatedWith")) { pstDescCompitem = Regex.Split(pstDescCompitem, "\";\"associatedWith")[0]; } } if (string.IsNullOrEmpty(LDS_Desc_AllComp)) { LDS_Desc_AllComp = pstTitlesitem + "-" + pstDescCompitem + " ++ "; } else { LDS_Desc_AllComp = LDS_Desc_AllComp + pstTitlesitem + "-" + pstDescCompitem + " ++ "; } } catch { } } } #endregion #region AccountType try { if (stringSource.Contains("has a Premium Account") || stringSource.Contains("Account Holder")) { AccountType = "Premium Account"; } else { AccountType = "Basic Account"; } } catch (Exception ex) { } #endregion Email #region FullUrl try { string[] UrlFull = System.Text.RegularExpressions.Regex.Split(Url, "&authType"); LDS_UserProfileLink = UrlFull[0]; LDS_UserProfileLink = Url; // LDS_UserProfileLink = stringSource.Substring(stringSource.IndexOf("canonicalUrl\":"), (stringSource.IndexOf(",", stringSource.IndexOf("canonicalUrl\":")) - stringSource.IndexOf("canonicalUrl\":"))).Replace("canonicalUrl\":", string.Empty).Replace("\\", string.Empty).Replace("\"", string.Empty).Replace(",", string.Empty).Replace(":", string.Empty).Trim(); } catch { } #endregion LDS_LoginID = SearchCriteria.LoginID; if (string.IsNullOrEmpty(firstname)) { firstname = "Linkedin Member"; } string endName = firstname + " " + lastname; //GroupStatus.GroupSpecMem.Add(GroupMemId, endName); if (firstname == string.Empty) firstname = "LinkedIn"; if (lastname == string.Empty) lastname = "Member"; if (Company == string.Empty) Company = "--"; if (titlecurrent == string.Empty) titlecurrent = "--"; if (companycurrent == string.Empty) companycurrent = "--"; if (LDS_Desc_AllComp == string.Empty) LDS_Desc_AllComp = "--"; if (LDS_BackGround_Summary == string.Empty) LDS_BackGround_Summary = "--"; if (Connection == string.Empty) Connection = "--"; if (recomandation == string.Empty) recomandation = "--"; if (Skill == string.Empty) Skill = "--"; if (LDS_Experience == string.Empty) LDS_Experience = "--"; if (EducationCollection == string.Empty) EducationCollection = "--"; if (groupscollectin == string.Empty) groupscollectin = "--"; if (USERemail == string.Empty) USERemail = "--"; if (LDS_UserContact == string.Empty) LDS_UserContact = "--"; if (LDS_PastTitles == string.Empty) LDS_PastTitles = "--"; if (AllComapny == string.Empty) AllComapny = "--"; if (location == string.Empty) location = "--"; if (country == string.Empty) country = "--"; if (Industry == string.Empty) Industry = "--"; if (Website == string.Empty) Website = "--"; string LDS_FinalData = TypeOfProfile + "," + LDS_UserProfileLink + "," + firstname + "," + lastname + "," + Company.Replace(",", ";") + "," + titlecurrent.Replace(",", ";") + "," + companycurrent.Replace(",", ";") + "," + CurrentCompSite.Replace(",", ";") + "," + LDS_BackGround_Summary.Replace(",", ";") + "," + Connection.Replace(",", ";") + "," + recomandation.Replace(",", string.Empty) + "," + Skill.Replace(",", ";") + "," + LDS_Experience.Replace(",", string.Empty) + "," + EducationCollection.Replace(",", ";") + "," + groupscollectin.Replace(",", ";") + "," + USERemail.Replace(",", ";") + "," + LDS_UserContact.Replace(",", ";") + "," + LDS_PastTitles + "," + AllComapny.Replace(",", ";") + "," + location.Replace(",", ";") + "," + country.Replace(",", ";") + "," + Industry.Replace(",", ";") + "," + Website.Replace(",", ";") + "," + LDS_LoginID + "," + AccountType; if (!string.IsNullOrEmpty(firstname)) { //Log("[ " + DateTime.Now + " ] => [ Data : " + LDS_FinalData + " ]"); } else { AddToLogger("[ " + DateTime.Now + " ] => [ No Data For URL : " + Url + " ]"); GlobusFileHelper.AppendStringToTextfileNewLineWithCarat(Url, Globals.DesktopFolder + "\\UnScrapedList.txt"); } if (SearchCriteria.starter) { //string tempFinalData = LDS_FinalData.Replace(";", "").Replace(LDS_UserProfileLink, "").Replace(TypeOfProfile, "").Replace(",", "").Replace(LDS_LoginID, "").Trim(); //if (!string.IsNullOrEmpty(tempFinalData)) { //if (CheckEmployeeScraper) //{ // string FileName = "CompanyEmployeeScraper"; // AppFileHelper.AddingLinkedInDataToCSVFileCompanyEmployeeScraper(LDS_FinalData, FileName); // return true; //} //else if (CampaignScraper) { AppFileHelper.AddingLinkedInDataToCSVFile(LDS_FinalData, FileName); return true; } //else //{ // AppFileHelper.AddingLinkedInDataToCSVFile(LDS_FinalData, SearchCriteria.FileName); // return true; //} } } } catch { } return false; }