public List <string> GetHrefsByTagAndAttributeName(string pageSrcHtml, string TagName, string className) { List <string> lstData = new List <string>(); try { bool success = false; string xHtml = string.Empty; Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); //*** Check DLL working or not ********************** success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return(null); } htmlToXml.Html = pageSrcHtml; //** Convert Data Html to XML ******************************************* xHtml = htmlToXml.ToXml(); //****************************************** Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); #region Data Save in list From using XML Tag and Attribut string DescriptionMain = string.Empty; string dataDescription = string.Empty; xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, TagName, "class", className); while ((xNode != null)) { //** Get Data Under Tag only Text Value********************************** dataDescription = xNode.GetXml();//.AccumulateTagContent("text", "script|style"); List <string> lstHrefs = GetHrefFromString(dataDescription); lstData.AddRange(lstHrefs);//lstData.Add(dataDescription); //** Get Data Under Tag All Html value * ********************************* //dataDescription = xNode.GetXml(); xBeginSearchAfter = xNode; xNode = xml.SearchForAttribute(xBeginSearchAfter, TagName, "class", className); } #endregion return(lstData); } catch (Exception) { return(lstData = null); } }
public List<string> GetDataTag(string pageSrcHtml, string TagName) { bool success = false; string xHtml = string.Empty; List<string> list = new List<string>(); try { Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return null; } htmlToXml.Html = pageSrcHtml; //xHtml contain xml data xHtml = htmlToXml.ToXml(); //****************************************** Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); xBeginSearchAfter = null; xNode = xml.SearchForTag(xBeginSearchAfter, TagName); while ((xNode != null)) { string TagText = xNode.AccumulateTagContent("text", "script|style"); list.Add(TagText); xBeginSearchAfter = xNode; xNode = xml.SearchForTag(xBeginSearchAfter, TagName); } //xHtml. } catch (Exception ex) { //GlobusLogHelper.log.Error("Error : " + ex.StackTrace); } return list; }
public List <string> GetDataTagDirectTagData(string pageSrcHtml, string TagName) { bool success = false; string xHtml = string.Empty; List <string> list = new List <string>(); Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return(null); } htmlToXml.Html = pageSrcHtml; //xHtml contain xml data xHtml = htmlToXml.ToXml(); //****************************************** Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); xBeginSearchAfter = null; xNode = xml.SearchForTag(xBeginSearchAfter, TagName); while ((xNode != null)) { string TagText = xNode.GetXml(); list.Add(TagText); xBeginSearchAfter = xNode; xNode = xml.SearchForTag(xBeginSearchAfter, TagName); } //xHtml. return(list); }
public string ConvertHtmlToXml(string PageSrcHtml) { //// Convert the HTML to XML: bool success = false; string xHtml = string.Empty; Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return(null); } htmlToXml.Html = PageSrcHtml; //xHtml contain xml data xHtml = htmlToXml.ToXml(); //Chilkat.Xml xml = new Chilkat.Xml(); //xml.LoadXml(xHtml); //xHtml. return(xHtml); }
public string ConvertHtmlToXml(string PageSrcHtml) { //// Convert the HTML to XML: bool success = false; string xHtml = string.Empty; Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return null; } htmlToXml.Html = PageSrcHtml; //xHtml contain xml data xHtml = htmlToXml.ToXml(); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); //xHtml. return xHtml; }
public void CrawlingPageDataSource(string Url, ref GlobusHttpHelper HttpHelper) { // if (SearchCriteria.starter) { // if (SearchCriteria.starter) { try { Log("[ " + DateTime.Now + " ] => [ Start Parsing Process ]"); #region Data Initialization string Industry = string.Empty; string URLprofile = string.Empty; string firstname = string.Empty; string lastname = string.Empty; string location = string.Empty; string country = string.Empty; string postal = string.Empty; string phone = string.Empty; string USERemail = string.Empty; string code = string.Empty; string education1 = string.Empty; string education2 = string.Empty; string titlecurrent = string.Empty; string companycurrent = string.Empty; string titlepast1 = string.Empty; string companypast1 = string.Empty; string titlepast2 = string.Empty; string html = string.Empty; string companypast2 = string.Empty; string titlepast3 = string.Empty; string companypast3 = string.Empty; string titlepast4 = string.Empty; string companypast4 = string.Empty; string Recommendations = string.Empty; string Connection = string.Empty; string Designation = string.Empty; string Website = string.Empty; string Contactsettings = string.Empty; string recomandation = string.Empty; string titleCurrenttitle = string.Empty; string titleCurrenttitle2 = string.Empty; string titleCurrenttitle3 = string.Empty; string titleCurrenttitle4 = string.Empty; string Skill = string.Empty; string TypeOfProfile = "Public1"; string Finaldata = string.Empty; #endregion #region LDS_DataInitialization string LDS_FirstName = string.Empty; string LDS_LastName = string.Empty; string LDS_UserProfileLink = string.Empty; string LDS_HeadLineTitle = string.Empty; string LDS_CurrentTitle = string.Empty; string LDS_PastTitles = string.Empty; string LDS_Loction = string.Empty; string LDS_Country = string.Empty; string LDS_Connection = string.Empty; string LDS_Recommendations = string.Empty; string LDS_SkillAndExpertise = string.Empty; string LDS_Education = string.Empty; string LDS_Experience = string.Empty; string LDS_ProfileType = "Public"; string LDS_Groups = string.Empty; string LDS_UserEmail = string.Empty; string LDS_UserContactNumber = string.Empty; string LDS_CurrentCompany = string.Empty; string LDS_PastCompany = string.Empty; string LDS_LoginID = string.Empty; string LDS_Websites = string.Empty; string LDS_Industry = string.Empty; #endregion #region Chilkat Initialization Chilkat.Http http = new Chilkat.Http(); ///Chilkat Http Request to be used in Http Post... Chilkat.HttpRequest req = new Chilkat.HttpRequest(); Chilkat.HtmlUtil htmlUtil = new Chilkat.HtmlUtil(); // Any string unlocks the component for the 1st 30-days. bool success = http.UnlockComponent("THEBACHttp_b3C9o9QvZQ06"); if (success != true) { Console.WriteLine(http.LastErrorText); return; } http.CookieDir = "memory"; http.SendCookies = true; http.SaveCookies = true; html = HttpHelper.getHtmlfromUrl1(new Uri(Url)); html = htmlUtil.EntityDecode(html); //// Convert the HTML to XML: Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); Chilkat.HtmlToXml htmlToXml1 = new Chilkat.HtmlToXml(); Chilkat.HtmlToXml htmlToXml2 = new Chilkat.HtmlToXml(); success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return; } string xHtml = null; string xHtml1 = null; //string xHtml2 = null; htmlToXml.Html = html; xHtml = htmlToXml.ToXml(); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); //// Iterate over all h1 tags: Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion #region for paRSING List<string> list = new List<string>(); List<string> Grouplist = new List<string>(); List<string> listtitle = new List<string>(); List<string> Currentlist = new List<string>(); List<string> Skilllst = new List<string>(); list.Clear(); //new parshing code List<string> TempFirstName = objChilkat.GetDataTagAttributewithId(html, "div", "name-container"); xBeginSearchAfter = null; xNode = xml.SearchForTag(xBeginSearchAfter, "dt"); Grouplist.Clear(); xBeginSearchAfter = null; #region parsergroup xNode = xml.SearchForAttribute(xBeginSearchAfter, "div", "class", "group-data"); while ((xNode != null)) { Finaldata = xNode.AccumulateTagContent("text", "/text"); Grouplist.Add(Finaldata); string[] tempC1 = Regex.Split(Finaldata, " at "); xBeginSearchAfter = xNode; xNode = xml.SearchForAttribute(xBeginSearchAfter, "div", "class", "group-data"); } int groupcounter = 0; string AllGRoup = string.Empty; foreach (string item in Grouplist) { if (item.Contains("Join")) { if (groupcounter == 0) { LDS_Groups = item; groupcounter++; } else { LDS_Groups = AllGRoup + ";" + item; } } } #endregion #region parserSkill xNode = xml.SearchForTag(xBeginSearchAfter, "dt"); Skilllst.Clear(); xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, "div", "id", "profile-skills"); while ((xNode != null)) { Finaldata = xNode.AccumulateTagContent("text", "/text"); if (Finaldata.Contains("extlib: _toggleclass")) { try { string[] Temp = Finaldata.Split(';'); LDS_SkillAndExpertise = Temp[4]; } catch { } } else { try { LDS_SkillAndExpertise = Finaldata.Replace("Skills & Expertise", " "); Skilllst.Add(Finaldata); } catch { } } xBeginSearchAfter = xNode; xNode = xml.SearchForAttribute(xBeginSearchAfter, "div", "id", "profile-skills"); } if (LDS_SkillAndExpertise.Contains(" Endorsements LI.i18n.register('section_skills_person_endorsed_tmpl")) { LDS_SkillAndExpertise = string.Empty; } Skilllst.Distinct(); #endregion #region UrlProfile try { if (html.Contains("webProfileURL")) { int FirstPointForProfileURL = html.IndexOf("webProfileURL"); string FirstSubStringForProfileURL = html.Substring(FirstPointForProfileURL); int SecondPointForProfileURL = FirstSubStringForProfileURL.IndexOf(">"); int ThirdPointForProfileURL = FirstSubStringForProfileURL.IndexOf("</a>"); string SecondSubStringForProfileURL = FirstSubStringForProfileURL.Substring(SecondPointForProfileURL, ThirdPointForProfileURL - SecondPointForProfileURL); LDS_UserProfileLink = SecondSubStringForProfileURL.Replace(">", string.Empty); //qm.AddProfileUrl(URLprofile, DateTime.Now.ToString(), "0"); } } catch (Exception ex) { Console.WriteLine(ex.Message); } try { string[] UrlFull = System.Text.RegularExpressions.Regex.Split(Url, "&authType"); LDS_UserProfileLink = UrlFull[0]; LDS_UserProfileLink = Url; } catch { } #endregion #region Connection if (html.Contains("overview-connections")) { try { Connection = html.Substring(html.IndexOf("leo-module mod-util connections"), 500); string[] Arr = Connection.Split('>'); string tempConnection = Arr[5].Replace("</strong", "").Replace(")</h3", "").Replace("(", ""); if (tempConnection.Length < 8) { LDS_Connection = tempConnection + "Connection"; } else { LDS_Connection = string.Empty; } } catch (Exception ex) { //overview-connections try { LDS_Connection = html.Substring(html.IndexOf("overview-connections"), 50); string[] Arr = Connection.Split('>'); string tempConnection = Arr[3].Replace("</strong", "").Replace(")</h3", "").Replace("(", ""); LDS_Connection = tempConnection + "Connection"; } catch { } } } #endregion #region Recommendation if (html.Contains("Recommendations")) { try { string[] rList = System.Text.RegularExpressions.Regex.Split(html, "Recommendations"); string[] R3List = rList[2].Split('\n'); string temprecomandation = R3List[4].Replace("</strong>", "").Replace("<strong>", ""); if (temprecomandation.Contains("recommended")) { LDS_Recommendations = temprecomandation; } else { LDS_Recommendations = ""; } } catch (Exception ex) { LDS_Recommendations = string.Empty; } } #endregion #region Websites if (html.Contains("websites")) { try { string websitedem = html.Substring(html.IndexOf("websites"), 500); string[] Arr = Regex.Split(websitedem, "href"); foreach (string item in Arr) { if (item.Contains("redir/redirect?url")) { string tempArr = item.Substring(item.IndexOf("name="), 50); string[] temarr = tempArr.Split('\n'); LDS_Websites = temarr[1]; } } } catch (Exception ex) { LDS_Websites = string.Empty; } } #endregion #region Getting Industry try { string Industrytemp = html.Substring(html.IndexOf("Find users in this industry"), 100); string[] TempIndustery = Industrytemp.Split('>'); LDS_Industry = TempIndustery[1].Replace("</strong", "").Replace("</a", ""); } catch (Exception ex) { Console.WriteLine(ex.Message); } #endregion #region Getting First Name try { if (html.Contains("given-name")) { int FirstPointForProfilename = html.IndexOf("given-name"); string FirstSubStringForProfilename = html.Substring(FirstPointForProfilename); int SecondPointForProfilename = FirstSubStringForProfilename.IndexOf(">"); int ThirdPointForProfilename = FirstSubStringForProfilename.IndexOf("</span>"); string SecondSubStringForProfilename = FirstSubStringForProfilename.Substring(SecondPointForProfilename, ThirdPointForProfilename - SecondPointForProfilename); LDS_FirstName = SecondSubStringForProfilename.Replace(">", string.Empty); } } catch (Exception ex) { Console.WriteLine(ex.Message); } #endregion #region LastName try { if (html.Contains("family-name")) { int FirstPointForProfilelastname = html.IndexOf("family-name"); string FirstSubStringForProfilelastname = html.Substring(FirstPointForProfilelastname); int SecondPointForProfilelastname = FirstSubStringForProfilelastname.IndexOf(">"); int ThirdPointForProfilelastname = FirstSubStringForProfilelastname.IndexOf("</span>"); string SecondSubStringForProfilelastname = FirstSubStringForProfilelastname.Substring(SecondPointForProfilelastname, ThirdPointForProfilelastname - SecondPointForProfilelastname); string templastname = SecondSubStringForProfilelastname.Replace(">", string.Empty); if (templastname.Contains(",")) { string[] arrylastname = templastname.Split(','); LDS_LastName = arrylastname[0]; } else { LDS_LastName = templastname; } } } catch (Exception ex) { Console.WriteLine(ex.Message); } #endregion #region Designation Company Current try { if (html.Contains("phonetic-full-name")) { int FirstPointForProfileCurrent = html.IndexOf("phonetic-full-name"); string FirstSubStringForProfileCurrent = html.Substring(FirstPointForProfileCurrent); int SecondPointForProfileCurrent = FirstSubStringForProfileCurrent.IndexOf("display:block"); int ThirdPointForProfileCurrent = FirstSubStringForProfileCurrent.IndexOf("</p>"); string SecondSubStringForProfileCurrent = FirstSubStringForProfileCurrent.Substring(SecondPointForProfileCurrent, ThirdPointForProfileCurrent - SecondPointForProfileCurrent); titlecurrent = SecondSubStringForProfileCurrent.Replace("\">", "").Replace("display:block", string.Empty).Replace("<strong class=\"highlight\"", string.Empty).Replace("</strong", string.Empty).Trim(); string[] tempCCurent = Regex.Split(titlecurrent, " at "); LDS_HeadLineTitle = titlecurrent.Replace(",", ";"); LDS_CurrentCompany = tempCCurent[1].Replace(",", ";"); } else if (html.Contains("<p class=\"title\"")) { LDS_HeadLineTitle = html.Substring(html.IndexOf("<p class=\"title\""), 150); string[] HeadLineTitle = LDS_HeadLineTitle.Split('>'); string tempHeadLineTitle = HeadLineTitle[1].Replace("\n", "").Replace(")</h3", "").Replace("</p", ""); LDS_HeadLineTitle = tempHeadLineTitle; try { string[] tempCCurent = Regex.Split(tempHeadLineTitle, " at "); LDS_HeadLineTitle = tempCCurent[0]; LDS_CurrentCompany = tempCCurent[1]; } catch { } } } catch (Exception ex) { Console.WriteLine(ex.Message); } #endregion #region Education try { if (html.Contains("summary-education")) { int FirstPointForProfileeducation1 = html.IndexOf("summary-education"); string FirstSubStringForProfileeducation1 = html.Substring(FirstPointForProfileeducation1); int SecondPointForProfileeducation1 = FirstSubStringForProfileeducation1.IndexOf("<li>"); int ThirdPointForProfileeducation1 = FirstSubStringForProfileeducation1.IndexOf("</li>"); string SecondSubStringForProfileeducation1 = FirstSubStringForProfileeducation1.Substring(SecondPointForProfileeducation1, ThirdPointForProfileeducation1 - SecondPointForProfileeducation1); education1 = SecondSubStringForProfileeducation1.Replace("<li>", string.Empty).Replace(",", string.Empty).Trim(); } } catch (Exception ex) { Console.WriteLine(ex.Message); } #endregion #region Country try { if (html.Contains("locality")) { int FirstPointForlocality = html.IndexOf("locality"); string FirstSubStringForlocality = html.Substring(FirstPointForlocality); int SecondPointForlocality = FirstSubStringForlocality.IndexOf("location"); int ThirdPointForlocality = FirstSubStringForlocality.IndexOf("</a>"); string SecondSubStringForlocality = FirstSubStringForlocality.Substring(SecondPointForlocality, ThirdPointForlocality - SecondPointForlocality); string temlocation = SecondSubStringForlocality.Replace("location", string.Empty).Replace(">", string.Empty).Replace('"', ' '); string[] temp = temlocation.Split(','); LDS_Loction = temp[0].Replace("<strong class= highlight", string.Empty).Replace("</strong", string.Empty); LDS_Country = temp[1].Replace("<strong class= highlight", string.Empty).Replace("</strong", string.Empty); // country = temp[1].Replace("</strong", string.Empty); } } catch (Exception ex) { Console.WriteLine(ex.Message); } #endregion #region User Email try { if (html.Contains("Email & Phone:")) { int FirstPointFortitlepast1 = html.IndexOf("abook-email"); string FirstSubStringFortitlepast1 = html.Substring(FirstPointFortitlepast1); int SecondPointFortitlepast1 = FirstSubStringFortitlepast1.IndexOf("<a"); int ThirdPointFortitlepast1 = FirstSubStringFortitlepast1.IndexOf("</a>"); string SecondSubStringFortitlepast1 = FirstSubStringFortitlepast1.Substring(SecondPointFortitlepast1, ThirdPointFortitlepast1 - SecondPointFortitlepast1); string[] tempEmail = SecondSubStringFortitlepast1.Split('>'); LDS_UserEmail = tempEmail[1]; } } catch (Exception ex) { Console.WriteLine(ex.Message); } #endregion #region Type Of profile try { if (html.Contains("profile-header")) { int FirstPointForProfileType = html.IndexOf("profile-header"); string FirstSubStringForProfileType = html.Substring(FirstPointForProfileType); int SecondPointForProfileType = FirstSubStringForProfileType.IndexOf("class=\"n fn\""); int ThirdPointForProfileType = FirstSubStringForProfileType.IndexOf("</span>"); string SecondSubStringForProfileType = FirstSubStringForProfileType.Substring(SecondPointForProfileType, ThirdPointForProfileType - SecondPointForProfileType); string[] tempProfileType = SecondSubStringForProfileType.Split('>'); string ProfileType = tempProfileType[1]; LDS_ProfileType = ProfileType; } //<h1><span id="name" class="n fn">Private</span> else if (html.Contains(" class=\"n fn\"")) { try { string ProfileTypetemp = html.Substring(html.IndexOf("class=\"n fn\""), 20); string[] TempProfileType = ProfileTypetemp.Split('>'); LDS_ProfileType = TempProfileType[1].Replace("</strong", "").Replace("</a", ""); } catch { } } if (LDS_ProfileType != "Public") { LDS_ProfileType = "Private"; } } catch (Exception ex) { Console.WriteLine(ex.Message); } #endregion #region PhonNumber try { if (html.Contains("<dt>Phone:</dt>")) { int FirstPointFortitlepast1 = html.IndexOf("profile-personal"); string FirstSubStringFortitlepast1 = html.Substring(FirstPointFortitlepast1); int SecondPointFortitlepast1 = FirstSubStringFortitlepast1.IndexOf("<p>"); int ThirdPointFortitlepast1 = FirstSubStringFortitlepast1.IndexOf("<span"); string SecondSubStringFortitlepast1 = FirstSubStringFortitlepast1.Substring(SecondPointFortitlepast1, ThirdPointFortitlepast1 - SecondPointFortitlepast1); LDS_UserContactNumber = SecondSubStringFortitlepast1.Replace("<p>", string.Empty); } } catch (Exception ex) { Console.WriteLine(ex.Message); } #endregion xNode = xml.SearchForTag(xBeginSearchAfter, "dt"); xBeginSearchAfter = xNode; list.Clear(); #endregion #region Regionfor PastCompney try { if (html.Contains("summary-past")) { int FirstPointForPasttitle = html.IndexOf("summary-past"); string FirstSubStringForPasttitle = html.Substring(FirstPointForPasttitle); int SecondPointForPasttitle = FirstSubStringForPasttitle.IndexOf("<li>"); int ThirdPointForPasttitle = FirstSubStringForPasttitle.IndexOf("summary-education"); string SecondSubStringForPasttitle = FirstSubStringForPasttitle.Substring(SecondPointForPasttitle, ThirdPointForPasttitle - SecondPointForPasttitle); string FirstSubStringForPasttitlelast = htmlUtil.EntityDecode(SecondSubStringForPasttitle); htmlToXml1.Html = FirstSubStringForPasttitlelast; xHtml1 = htmlToXml1.ToXml(); Chilkat.Xml xml1 = new Chilkat.Xml(); xml1.LoadXml(xHtml1); //// Iterate over all h1 tags: Chilkat.Xml xNode1 = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter1 = default(Chilkat.Xml); list.Clear(); string[] tempC1 = null; xNode1 = xml1.SearchForTag(xBeginSearchAfter1, "li"); while ((xNode1 != null)) { Finaldata = xNode1.AccumulateTagContent("text", "/text"); listtitle.Add(Finaldata); // list.Add(Finaldata); try { tempC1 = Regex.Split(Finaldata, " at "); } catch { } if (tempC1 != null) { try { list.Add(tempC1[1]); } catch { } } xNode1 = xml1.SearchForTag(xBeginSearchAfter1, "li"); xBeginSearchAfter1 = xNode1; } if (listtitle.Count > 0 || list.Count > 0) { try { titlepast1 = listtitle[0] != null ? listtitle[0] : string.Empty; titlepast2 = listtitle[1] != null ? listtitle[1] : string.Empty; titlepast3 = listtitle[2] != null ? listtitle[2] : string.Empty; titlepast4 = listtitle[3] != null ? listtitle[3] : string.Empty; } catch { } try { companypast1 = list[0] != null ? list[0] : string.Empty; companypast2 = list[1] != null ? list[1] : string.Empty; companypast3 = list[2] != null ? list[2] : string.Empty; companypast4 = list[3] != null ? list[3] : string.Empty; } catch { } } } } catch { }; list.Clear(); #endregion string companyCurrenttitle1 = string.Empty; string companyCurrenttitle2 = string.Empty; string companyCurrenttitle3 = string.Empty; string companyCurrenttitle4 = string.Empty; #region Regionfor summary-current try { if (html.Contains("summary-current")) { int FirstPointForCurrenttitle = html.IndexOf("summary-current"); string FirstSubStringForCurrenttitle = html.Substring(FirstPointForCurrenttitle); int SecondPointForCurrenttitle = FirstSubStringForCurrenttitle.IndexOf("<li>"); int ThirdPointForCurrenttitle = FirstSubStringForCurrenttitle.IndexOf("summary-past"); string SecondSubStringForCurrenttitle = FirstSubStringForCurrenttitle.Substring(SecondPointForCurrenttitle, ThirdPointForCurrenttitle - SecondPointForCurrenttitle); string FirstSubStringForCurrenttitlelast = htmlUtil.EntityDecode(SecondSubStringForCurrenttitle); htmlToXml1.Html = FirstSubStringForCurrenttitlelast; xHtml1 = htmlToXml1.ToXml(); Chilkat.Xml xml1 = new Chilkat.Xml(); xml1.LoadXml(xHtml1); //// Iterate over all h1 tags: Chilkat.Xml xNode1 = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter1 = default(Chilkat.Xml); Currentlist.Clear(); list.Clear(); string[] tempC1 = null; xNode1 = xml1.SearchForTag(xBeginSearchAfter1, "li"); while ((xNode1 != null)) { Finaldata = xNode1.AccumulateTagContent("text", "/text"); Currentlist.Add(Finaldata); // list.Add(Finaldata); try { tempC1 = Regex.Split(Finaldata, " at "); } catch { } if (tempC1 != null) { try { list.Add(tempC1[1]); } catch { } } xNode1 = xml1.SearchForTag(xBeginSearchAfter1, "li"); xBeginSearchAfter1 = xNode1; } if (Currentlist.Count > 0 || list.Count > 0) { try { titleCurrenttitle = Currentlist[0] != null ? Currentlist[0] : string.Empty; titleCurrenttitle2 = Currentlist[1] != null ? Currentlist[1] : string.Empty; titleCurrenttitle3 = Currentlist[2] != null ? Currentlist[2] : string.Empty; titleCurrenttitle4 = Currentlist[3] != null ? Currentlist[3] : string.Empty; } catch { } try { companyCurrenttitle1 = list[0] != null ? list[0] : string.Empty; companyCurrenttitle2 = list[1] != null ? list[1] : string.Empty; companyCurrenttitle3 = list[2] != null ? list[2] : string.Empty; companyCurrenttitle4 = list[3] != null ? list[3] : string.Empty; } catch { } } } } catch { }; list.Clear(); #endregion #region RegionForEDUCATION try { if (html.Contains("summary-education")) { int FirstPointForEDUCATION = html.IndexOf("summary-education"); string FirstSubStringForEDUCATION = html.Substring(FirstPointForEDUCATION); int SecondPointForEDUCATION = FirstSubStringForEDUCATION.IndexOf("<li>"); int ThirdPointForEDUCATION = FirstSubStringForEDUCATION.IndexOf("</ul>"); string SecondSubStringForEDUCATION = FirstSubStringForEDUCATION.Substring(SecondPointForEDUCATION, ThirdPointForEDUCATION - SecondPointForEDUCATION); //string tempEDu = SecondSubStringForEDUCATION.Replace("<li>", string.Empty).Replace("</li>", string.Empty).Replace(" ", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Trim(); string temptg = SecondSubStringForEDUCATION.Replace("<li>", ""); string[] templis6t = temptg.Split('/'); education1 = templis6t[0].Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("<", string.Empty).Replace("span>", string.Empty).Replace(",", string.Empty).Trim(); education2 = templis6t[1].Replace("\n", string.Empty).Replace("\t", string.Empty).Replace("li>", string.Empty).Replace("<", string.Empty).Replace("span>", string.Empty).Replace(",", string.Empty).Trim(); } } catch { }; list.Clear(); #endregion string GroupPastJob = string.Empty; string GroupEduction = string.Empty; LDS_PastTitles = titlepast1 + ";" + titlepast3; LDS_PastCompany = companypast1 + ";" + companypast3; LDS_Education = education1 + ";" + education2; LDS_CurrentTitle = titleCurrenttitle; LDS_LoginID = _UserName;//SearchCriteria.LoginID; //"ProfileType" + "," + "UserProfileLink" + "," + "FirstName" + "," + "LastName" + "," + "HeadLineTitle" + "," + "CurrentTitle " + "," + "Company" + "," + "Connection" + "," + "Recommendations " + "," + "SkillAndExpertise " + "," + "Experience " + "," + " Education" + "," + "Groups" + "," + "UserEmail" + "," + "UserContactNumbe" + "," + "PastTitles" + "," + "PastCompany" + "," + "Loction" + "," + "Country" + "," + "titlepast3" + "," + "companypast3" + "," + "titlepast4" + "," + "companypast4" + ","; string LDS_FinalData = LDS_ProfileType.Replace(",", ";") + "," + LDS_UserProfileLink.Replace(",", ";") + "," + LDS_FirstName.Replace(",", ";") + "," + LDS_LastName.Replace(",", ";") + "," + LDS_HeadLineTitle.Replace(",", ";") + "," + LDS_CurrentTitle.Replace(",", ";") + "," + LDS_CurrentCompany.Replace(",", ";") + "," + LDS_Connection.Replace(",", ";") + "," + LDS_Recommendations.Replace(",", ";") + "," + LDS_SkillAndExpertise.Replace(",", ";") + "," + LDS_Experience.Replace(",", ";") + "," + LDS_Education.Replace(",", ";") + "," + LDS_Groups.Replace(",", ";") + "," + LDS_UserEmail.Replace(",", ";") + "," + LDS_UserContactNumber.Replace(",", ";") + "," + LDS_PastTitles.Replace(",", ";") + "," + LDS_PastCompany.Replace(",", ";") + "," + LDS_Loction.Replace(",", ";") + "," + LDS_Country.Replace(",", ";") + "," + LDS_Industry.Replace(",", ";") + "," + LDS_Websites.Replace(",", ";") + "," + LDS_LoginID.Replace(",", ";") + ","; if (LDS_FinalData.Contains("<strong class=\"highlight\"") || LDS_FinalData.Contains("<span class=\"full-name\"") || LDS_FinalData.Contains("<strong class=\"highlight\"") || LDS_FinalData.Contains("overview-connections\">")) { LDS_FinalData = LDS_FinalData.Replace("<span class=\"full-name\"", "").Replace("\n", "").Replace("<strong class=\"highlight\"", "").Replace("overview-connections\">", "").Replace("</strong>", "").Replace("<strong>", ""); } if (!string.IsNullOrEmpty(LDS_FirstName) || !string.IsNullOrEmpty(LDS_FirstName)) { Log(LDS_FinalData); } // if (SearchCriteria.starter) { string tempFinalData = LDS_FinalData.Replace(";", "").Replace(LDS_UserProfileLink, "").Replace("Public", "").Replace(",", "").Replace(LDS_LoginID, "").Trim(); if (!string.IsNullOrEmpty(tempFinalData)) { //AppFileHelper.AddingLinkedInDataToCSVFile(LDS_FinalData, Globals.path_ScrappedMembersFromGroup); string CSVHeader = "ProfileType" + "," + "UserProfileLink" + "," + "FirstName" + "," + "LastName" + "," + "HeadLineTitle" + "," + "Current Title " + "," + "Current Company" + "," + "Connection" + "," + "Recommendations " + "," + "SkillAndExpertise " + "," + "Experience " + "," + " Education" + "," + "Groups" + "," + "UserEmail" + "," + "UserContactNumber" + "," + "PastTitles" + "," + "PastCompany" + "," + "Location" + "," + "Country" + "," + "Industry" + "," + "WebSites" + "," + "LinkedInLoginID" + ","; string CSV_Content = TypeOfProfile.Replace(",", ";") + "," + LDS_UserProfileLink.Replace(",", ";") + "," + LDS_FirstName.Replace(",", ";") + "," + LDS_LastName.Replace(",", ";") + "," + LDS_HeadLineTitle.Replace(",", ";") + "," + LDS_CurrentTitle.Replace(",", ";") + "," + LDS_CurrentCompany.Replace(",", ";") + "," + LDS_Connection.Replace(",", ";") + "," + LDS_Recommendations.Replace(",", ";") + "," + LDS_SkillAndExpertise.Replace(",", ";") + "," + LDS_Experience.Replace(",", ";") + "," + LDS_Education.Replace(",", ";") + "," + LDS_Groups.Replace(",", ";") + "," + LDS_UserEmail.Replace(",", ";") + "," + LDS_UserContactNumber.Replace(",", ";") + "," + LDS_PastTitles.Replace(",", ";") + "," + LDS_PastCompany.Replace(",", ";") + "," + LDS_Loction.Replace(",", ";") + "," + LDS_Country.Replace(",", ";") + "," + LDS_Industry.Replace(",", ";") + "," + LDS_Websites.Replace(",", ";") + "," + LDS_LoginID.Replace(",", ";"); //string CSV_Content = TypeOfProfile + "," + LDS_UserProfileLink + "," + firstname + "," + lastname + "," + Company.Replace(",", ";") + "," + titlecurrent.Replace(",", ";") + "," + companycurrent.Replace(",", ";") + "," + Connection.Replace(",", ";") + "," + recomandation.Replace(",", string.Empty) + "," + Skill.Replace(",", ";") + "," + LDS_Experience.Replace(",", string.Empty) + "," + EducationCollection.Replace(",", ";") + "," + groupscollectin.Replace(",", ";") + "," + USERemail.Replace(",", ";") + "," + LDS_UserContact.Replace(",", ";") + "," + LDS_PastTitles + "," + AllComapny.Replace(",", ";") + "," + country.Replace(",", ";") + "," + location.Replace(",", ";") + "," + Industry.Replace(",", ";") + "," + Website.Replace(",", ";") + "," + LDS_LoginID + ",";// +TypeOfProfile + ","; CSVUtilities.ExportDataCSVFile(CSVHeader, CSV_Content, Globals.path_ScrappedMembersFromGroup); Log("[ " + DateTime.Now + " ] => [ Data Saved In CSV File With URL >>> " + LDS_UserProfileLink + " ]"); } //if (!string.IsNullOrEmpty(LDS_FirstName) || !string.IsNullOrEmpty(LDS_FirstName)) //{ // AppFileHelper.AddingLinkedInDataToCSVFile(LDS_FinalData, SearchCriteria.FileName); //} } } catch (Exception ex) { }; } } }
public List<string> GetHrefsByTagAndAttributeName(string pageSrcHtml, string TagName, string className) { List<string> lstData = new List<string>(); try { bool success = false; string xHtml = string.Empty; Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); //*** Check DLL working or not ********************** success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return null; } htmlToXml.Html = pageSrcHtml; //** Convert Data Html to XML ******************************************* xHtml = htmlToXml.ToXml(); //****************************************** Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); #region Data Save in list From using XML Tag and Attribut string DescriptionMain = string.Empty; string dataDescription = string.Empty; xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, TagName, "class", className); while ((xNode != null)) { //** Get Data Under Tag only Text Value********************************** dataDescription = xNode.GetXml();//.AccumulateTagContent("text", "script|style"); List<string> lstHrefs = GetHrefFromString(dataDescription); lstData.AddRange(lstHrefs);//lstData.Add(dataDescription); //** Get Data Under Tag All Html value * ********************************* //dataDescription = xNode.GetXml(); xBeginSearchAfter = xNode; xNode = xml.SearchForAttribute(xBeginSearchAfter, TagName, "class", className); } #endregion return lstData; } catch (Exception) { return lstData = null; } }
public string GetDataWithTagValueByTagAndAttributeNameWithId(string pageSrcHtml, string TagName, string AttributeName) { string dataDescription = string.Empty; try { bool success = false; string xHtml = string.Empty; Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); //*** Check DLL working or not ********************** success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return(null); } htmlToXml.Html = pageSrcHtml; //** Convert Data Html to XML ******************************************* xHtml = htmlToXml.ToXml(); //****************************************** Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); #region Data Save in list From using XML Tag and Attribut string DescriptionMain = string.Empty; string dataDescriptionValue = string.Empty; xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, TagName, "id", AttributeName); while ((xNode != null)) { //** Get Data Under Tag only Text Value********************************** dataDescription = xNode.GetXml();//.AccumulateTagContent("text", "script|style"); dataDescriptionValue = dataDescriptionValue + dataDescription; // string text = xNode.AccumulateTagContent("text", "script|style"); // lstData.Add(text); // //** Get Data Under Tag All Html value * ********************************* // //dataDescription = xNode.GetXml(); xBeginSearchAfter = xNode; xNode = xml.SearchForAttribute(xBeginSearchAfter, TagName, "id", AttributeName); //if (dataDescription.Length > 500) //{ // break; //} } #endregion return(dataDescriptionValue); } catch (Exception) { return(dataDescription = null); } }
public string GetDataWithTagValueByTagAndAttributeNameWithId(string pageSrcHtml, string TagName, string AttributeName) { string dataDescription = string.Empty; try { bool success = false; string xHtml = string.Empty; Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); //*** Check DLL working or not ********************** success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return null; } htmlToXml.Html = pageSrcHtml; //** Convert Data Html to XML ******************************************* xHtml = htmlToXml.ToXml(); //****************************************** Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); #region Data Save in list From using XML Tag and Attribut string DescriptionMain = string.Empty; string dataDescriptionValue = string.Empty; xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, TagName, "id", AttributeName); while ((xNode != null)) { //** Get Data Under Tag only Text Value********************************** dataDescription = xNode.GetXml();//.AccumulateTagContent("text", "script|style"); dataDescriptionValue = dataDescriptionValue + dataDescription; // string text = xNode.AccumulateTagContent("text", "script|style"); // lstData.Add(text); // //** Get Data Under Tag All Html value * ********************************* // //dataDescription = xNode.GetXml(); xBeginSearchAfter = xNode; xNode = xml.SearchForAttribute(xBeginSearchAfter, TagName, "id", AttributeName); //if (dataDescription.Length > 500) //{ // break; //} } #endregion return dataDescriptionValue; } catch (Exception) { return dataDescription = null; } }
public List<string> GetDataTagDirectTagData(string pageSrcHtml, string TagName) { bool success = false; string xHtml = string.Empty; List<string> list = new List<string>(); Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return null; } htmlToXml.Html = pageSrcHtml; //xHtml contain xml data xHtml = htmlToXml.ToXml(); //****************************************** Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); xBeginSearchAfter = null; xNode = xml.SearchForTag(xBeginSearchAfter, TagName); while ((xNode != null)) { string TagText = xNode.GetXml(); list.Add(TagText); xBeginSearchAfter = xNode; xNode = xml.SearchForTag(xBeginSearchAfter, TagName); } //xHtml. return list; }
public void getmentions() { if (!IsLoggedIn) { Login(); } if (IsNotSuspended) { string pageSource = globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + Screen_name), "", ""); string[] href = Regex.Split(pageSource, "href=\"/" + Screen_name + "/status/"); href = href.Skip(1).ToArray(); foreach (string abc in href) { if (abc.Contains("tweet-timestamp js-permalink js-nav") && abc.Contains("js-tweet-text")) { string statusid = string.Empty; try { int startindex = abc.IndexOf("\""); string start = abc.Substring(0, startindex); statusid = start; } catch (Exception ex) { } string StatusPageSource = globusHttpHelper.getHtmlfromUrl(new Uri("https://twitter.com/" + Screen_name + "/status/" + statusid), "", ""); string[] getTweets = Regex.Split(StatusPageSource, "simple-tweet tweet js-stream-tweet"); getTweets = getTweets.Skip(1).ToArray(); foreach (string tweets in getTweets) { string TweetText = string.Empty; string From_user_Screen_name = string.Empty; string From_user_id = string.Empty; #region Convert HTML to XML Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); bool success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return; } string xHtml = null; htmlToXml.Html = tweets; //xHtml contain xml data xHtml = htmlToXml.ToXml(); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); //xHtml. //// Iterate over all h1 tags: Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); #endregion xNode = null; xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, "p", "class", "js-tweet-text"); while ((xNode != null)) { TweetText = xNode.AccumulateTagContent("text", "script|style"); break; } try { int startindex = tweets.IndexOf("data-screen-name"); string start = tweets.Substring(startindex).Replace("data-screen-name=\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("screen_name", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); From_user_Screen_name = end; } catch (Exception ex) { //Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user --> " + ex.Message, Globals.Path_TwitterDataScrapper); //Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user --> " + ex.Message, Globals.Path_TwtErrorLogs); } try { int startindex = tweets.IndexOf("data-user-id"); string start = tweets.Substring(startindex).Replace("data-user-id=\"", ""); int endIndex = start.IndexOf("\""); string end = start.Substring(0, endIndex).Replace("screen_name", "").Replace("\"", "").Replace(":", "").Replace("{", "").Replace("}", "").Replace("]", ""); From_user_id = end; } catch (Exception ex) { //Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine(DateTime.Now + " --> Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user --> " + ex.Message, Globals.Path_TwitterDataScrapper); //Globussoft.GlobusFileHelper.AppendStringToTextfileNewLine("Error --> GetPhotoFromUsername() -- " + keyword + " -- from_user --> " + ex.Message, Globals.Path_TwtErrorLogs); } string strQuery = "INSERT INTO tb_ReplyCampaign (TweetId , Username , ReplyUserName , ReplyUserId , TweetText , Reply) VALUES ('" + statusid + "' , '" + Username + "' , '" + From_user_Screen_name + "' , '" + From_user_id + "' , '" + TweetText + "' , '" + TweetText + "')"; DataBaseHandler.InsertQuery(strQuery, "tb_ReplyCampaign"); } } } } }
public bool CheckAttributeexsist(string pageSrcHtml, string TagName, string AttributeName) { bool IsContain = false; try { bool success = false; string xHtml = string.Empty; Chilkat.HtmlToXml htmlToXml = new Chilkat.HtmlToXml(); //*** Check DLL working or not ********************** success = htmlToXml.UnlockComponent("THEBACHtmlToXml_7WY3A57sZH3O"); if ((success != true)) { Console.WriteLine(htmlToXml.LastErrorText); return IsContain; } htmlToXml.Html = pageSrcHtml; //** Convert Data Html to XML ******************************************* xHtml = htmlToXml.ToXml(); //****************************************** Chilkat.Xml xNode = default(Chilkat.Xml); Chilkat.Xml xBeginSearchAfter = default(Chilkat.Xml); Chilkat.Xml xml = new Chilkat.Xml(); xml.LoadXml(xHtml); #region Data Save in list From using XML Tag and Attribut string DescriptionMain = string.Empty; string dataDescription = string.Empty; xBeginSearchAfter = null; xNode = xml.SearchForAttribute(xBeginSearchAfter, TagName, "class", AttributeName); while ((xNode != null)) { IsContain = true; return IsContain; } #endregion } catch (Exception) { IsContain = false; } return IsContain; }