private static MagazinesGratuits_PostDetail_v2 GetData(WebResult webResult) { XXElement xeSource = webResult.Http.zGetXDocument().zXXElement(); MagazinesGratuits_PostDetail_v2 data = new MagazinesGratuits_PostDetail_v2(); data.SourceUrl = webResult.WebRequest.HttpRequest.Url; data.LoadFromWebDate = webResult.LoadFromWebDate; //data.Id = _GetPostDetailKey(webResult.WebRequest.HttpRequest); data.Key = _GetPostDetailKey(webResult.WebRequest.HttpRequest); XXElement xePost = xeSource.XPathElement("//table[@id='layout']//div[@id='content']//div[@class='post']"); XXElement xe = xePost.XPathElement(".//table[@id='post-head']"); string[] dates = xe.XPathElement(".//td[@id='head-date']").DescendantTexts().Select(DownloadPrint.Trim).ToArray(); data.PostCreationDate = GetDate(dates, __lastPostDate); if (data.PostCreationDate != null) __lastPostDate = new Date(data.PostCreationDate.Value); if (__trace) pb.Trace.WriteLine("post creation date {0} - {1}", data.PostCreationDate, dates.zToStringValues()); //data.Title = xePost.XPathValue(".//div[@class='title']//a//text()", DownloadPrint.TrimFunc1); data.Title = xePost.XPathValue(".//div[@class='title']//a//text()").zFunc(DownloadPrint.ReplaceChars).zFunc(DownloadPrint.Trim); PrintTitleInfos titleInfos = DownloadPrint.PrintTextValuesManager.ExtractTitleInfos(data.Title); if (titleInfos.foundInfo) { data.OriginalTitle = data.Title; data.Title = titleInfos.title; data.Infos.SetValues(titleInfos.infos); } // Ebooks en Epub / Livre //data.Category = xePost.DescendantTextList(".//div[@class='postdata']//span[@class='category']//a").Select(DownloadPrint.TrimFunc1).zToStringValues("/"); data.Category = xePost.XPathElements(".//div[@class='postdata']//span[@class='category']//a").DescendantTexts().Select(DownloadPrint.Trim).zToStringValues("/"); data.PrintType = GetPrintType(data.Category); //pb.Trace.WriteLine("category \"{0}\" printType {1}", category, data.printType); xe = xePost.XPathElement(".//div[@class='entry']"); data.Images = new WebImage[] { new WebImage(zurl.GetUrl(data.SourceUrl, xe.XPathValue("div[starts-with(@class, 'post-views')]/following-sibling::h3/following-sibling::p/img/@src"))) }; // force load image to get image width and height if (webResult.WebRequest.LoadImage) data.Images = DownloadPrint.LoadImages(data.Images).ToArray(); // get infos, description, language, size, nbPages // xe.DescendantTextList(".//p") PrintTextValues textValues = DownloadPrint.PrintTextValuesManager.GetTextValues( xe.XPathElements(".//p").DescendantTexts( node => { if (node is XText) { string text = ((XText)node).Value.Trim(); //if (text.StartsWith("Lien Direct", StringComparison.InvariantCultureIgnoreCase)) if (text.StartsWith("lien ", StringComparison.InvariantCultureIgnoreCase)) return XNodeFilter.Stop; } if (node is XElement) { XElement xe2 = (XElement)node; if (xe2.Name == "p" && xe2.zAttribValue("class") == "submeta") return XNodeFilter.Stop; } return XNodeFilter.SelectNode; } ).Select(DownloadPrint.ReplaceChars).Select(DownloadPrint.TrimWithoutColon), data.Title); data.Description = textValues.description; //data.Language = textValues.language; //data.Size = textValues.size; //data.NbPages = textValues.nbPages; data.Infos.SetValues(textValues.infos); //data.DownloadLinks = xe.DescendantNodes( // node => // { // if (!(node is XElement)) // return true; // XElement xe2 = (XElement)node; // if (xe2.Name != "p") // return true; // XAttribute xa = xe2.Attribute("class"); // if (xa == null) // return true; // if (xa.Value != "submeta") // return true; // return false; // }, // node => node is XElement && ((XElement)node).Name == "a") // .Select(node => ((XElement)node).Attribute("href").Value).ToArray(); data.DownloadLinks = xe.DescendantNodes( node => { if (!(node is XElement)) return XNodeFilter.DontSelectNode; XElement xe2 = (XElement)node; if (xe2.Name == "a") return XNodeFilter.SelectNode; if (xe2.Name != "p") return XNodeFilter.DontSelectNode; XAttribute xa = xe2.Attribute("class"); if (xa == null) return XNodeFilter.DontSelectNode; if (xa.Value != "submeta") return XNodeFilter.DontSelectNode; //return XNodeFilter.SkipNode; return XNodeFilter.Stop; }) .Select(node => ((XElement)node).Attribute("href").Value).ToArray(); //// <div id="postlist" class="postlist restrain"> //xe = xePost.XPathElement(".//div[@id='postlist']"); //// Aujourd'hui, 07h32 - Aujourd'hui, 10h51 - Hier, 12h55 - 22/02/2014, 21h09 ////string date = xe.DescendantTextList(".//div[@class='posthead']//text()", nodeFilter: node => node.zGetName() != "a").zToStringValues(""); //XXElement xe2 = xe.XPathElement(".//div[@class='posthead']"); //string date = xe2.DescendantTextList(nodeFilter: node => node.zGetName() != "a").zToStringValues(""); //date = date.Replace('\xA0', ' '); //data.PostCreationDate = zdate.ParseDateTimeLikeToday(date, webResult.LoadFromWebDate, @"d/M/yyyy, HH\hmm", @"d-M-yyyy, HH\hmm"); //if (data.PostCreationDate == null) // pb.Trace.WriteLine("unknow post creation date \"{0}\"", date); //data.PostAuthor = xe.XPathValue(".//div[@class='userinfo']//a//text()", DownloadPrint.TrimFunc1); //// <div class="postbody"> //xe = xePost.XPathElement(".//div[@class='postbody']//div[@class='content']//blockquote/div"); //data.Images = xe.XPathImages(xeImg => new UrlImage(zurl.GetUrl(data.SourceUrl, xeImg.zAttribValue("src")))).ToArray(); //// get infos, description, language, size, nbPages //PrintTextValues textValues = DownloadPrint.PrintTextValuesManager.GetTextValues(xe.DescendantTextList(nodeFilter: node => !(node is XElement) || ((XElement)node).Name != "a"), data.Title); //data.Description = textValues.description; //data.Language = textValues.language; //data.Size = textValues.size; //data.NbPages = textValues.nbPages; //data.Infos.SetValues(textValues.infos); //data.DownloadLinks = xe.XPathValues(".//a/@href"); if (__trace) pb.Trace.WriteLine(data.zToJson()); return data; }
private static MagazinesGratuits_PostDetail_v2 GetData(WebResult webResult) { XXElement xeSource = webResult.Http.zGetXDocument().zXXElement(); MagazinesGratuits_PostDetail_v2 data = new MagazinesGratuits_PostDetail_v2(); data.SourceUrl = webResult.WebRequest.HttpRequest.Url; data.LoadFromWebDate = webResult.LoadFromWebDate; //data.Id = _GetPostDetailKey(webResult.WebRequest.HttpRequest); data.Key = _GetPostDetailKey(webResult.WebRequest.HttpRequest); XXElement xePost = xeSource.XPathElement("//table[@id='layout']//div[@id='content']//div[@class='post']"); XXElement xe = xePost.XPathElement(".//table[@id='post-head']"); string[] dates = xe.XPathElement(".//td[@id='head-date']").DescendantTexts().Select(DownloadPrint.Trim).ToArray(); data.PostCreationDate = GetDate(dates, __lastPostDate); if (data.PostCreationDate != null) { __lastPostDate = new Date(data.PostCreationDate.Value); } if (__trace) { pb.Trace.WriteLine("post creation date {0} - {1}", data.PostCreationDate, dates.zToStringValues()); } //data.Title = xePost.XPathValue(".//div[@class='title']//a//text()", DownloadPrint.TrimFunc1); data.Title = xePost.XPathValue(".//div[@class='title']//a//text()").zFunc(DownloadPrint.ReplaceChars).zFunc(DownloadPrint.Trim); PrintTitleInfos titleInfos = DownloadPrint.PrintTextValuesManager.ExtractTitleInfos(data.Title); if (titleInfos.foundInfo) { data.OriginalTitle = data.Title; data.Title = titleInfos.title; data.Infos.SetValues(titleInfos.infos); } // Ebooks en Epub / Livre //data.Category = xePost.DescendantTextList(".//div[@class='postdata']//span[@class='category']//a").Select(DownloadPrint.TrimFunc1).zToStringValues("/"); data.Category = xePost.XPathElements(".//div[@class='postdata']//span[@class='category']//a").DescendantTexts().Select(DownloadPrint.Trim).zToStringValues("/"); data.PrintType = GetPrintType(data.Category); //pb.Trace.WriteLine("category \"{0}\" printType {1}", category, data.printType); xe = xePost.XPathElement(".//div[@class='entry']"); data.Images = new WebImage[] { new WebImage(zurl.GetUrl(data.SourceUrl, xe.XPathValue("div[starts-with(@class, 'post-views')]/following-sibling::h3/following-sibling::p/img/@src"))) }; // force load image to get image width and height if (webResult.WebRequest.LoadImage) { data.Images = DownloadPrint.LoadImages(data.Images).ToArray(); } // get infos, description, language, size, nbPages // xe.DescendantTextList(".//p") PrintTextValues textValues = DownloadPrint.PrintTextValuesManager.GetTextValues( xe.XPathElements(".//p").DescendantTexts( node => { if (node is XText) { string text = ((XText)node).Value.Trim(); //if (text.StartsWith("Lien Direct", StringComparison.InvariantCultureIgnoreCase)) if (text.StartsWith("lien ", StringComparison.InvariantCultureIgnoreCase)) { return(XNodeFilter.Stop); } } if (node is XElement) { XElement xe2 = (XElement)node; if (xe2.Name == "p" && xe2.zAttribValue("class") == "submeta") { return(XNodeFilter.Stop); } } return(XNodeFilter.SelectNode); } ).Select(DownloadPrint.ReplaceChars).Select(DownloadPrint.TrimWithoutColon), data.Title); data.Description = textValues.description; //data.Language = textValues.language; //data.Size = textValues.size; //data.NbPages = textValues.nbPages; data.Infos.SetValues(textValues.infos); //data.DownloadLinks = xe.DescendantNodes( // node => // { // if (!(node is XElement)) // return true; // XElement xe2 = (XElement)node; // if (xe2.Name != "p") // return true; // XAttribute xa = xe2.Attribute("class"); // if (xa == null) // return true; // if (xa.Value != "submeta") // return true; // return false; // }, // node => node is XElement && ((XElement)node).Name == "a") // .Select(node => ((XElement)node).Attribute("href").Value).ToArray(); data.DownloadLinks = xe.DescendantNodes( node => { if (!(node is XElement)) { return(XNodeFilter.DontSelectNode); } XElement xe2 = (XElement)node; if (xe2.Name == "a") { return(XNodeFilter.SelectNode); } if (xe2.Name != "p") { return(XNodeFilter.DontSelectNode); } XAttribute xa = xe2.Attribute("class"); if (xa == null) { return(XNodeFilter.DontSelectNode); } if (xa.Value != "submeta") { return(XNodeFilter.DontSelectNode); } //return XNodeFilter.SkipNode; return(XNodeFilter.Stop); }) .Select(node => ((XElement)node).Attribute("href").Value).ToArray(); //// <div id="postlist" class="postlist restrain"> //xe = xePost.XPathElement(".//div[@id='postlist']"); //// Aujourd'hui, 07h32 - Aujourd'hui, 10h51 - Hier, 12h55 - 22/02/2014, 21h09 ////string date = xe.DescendantTextList(".//div[@class='posthead']//text()", nodeFilter: node => node.zGetName() != "a").zToStringValues(""); //XXElement xe2 = xe.XPathElement(".//div[@class='posthead']"); //string date = xe2.DescendantTextList(nodeFilter: node => node.zGetName() != "a").zToStringValues(""); //date = date.Replace('\xA0', ' '); //data.PostCreationDate = zdate.ParseDateTimeLikeToday(date, webResult.LoadFromWebDate, @"d/M/yyyy, HH\hmm", @"d-M-yyyy, HH\hmm"); //if (data.PostCreationDate == null) // pb.Trace.WriteLine("unknow post creation date \"{0}\"", date); //data.PostAuthor = xe.XPathValue(".//div[@class='userinfo']//a//text()", DownloadPrint.TrimFunc1); //// <div class="postbody"> //xe = xePost.XPathElement(".//div[@class='postbody']//div[@class='content']//blockquote/div"); //data.Images = xe.XPathImages(xeImg => new UrlImage(zurl.GetUrl(data.SourceUrl, xeImg.zAttribValue("src")))).ToArray(); //// get infos, description, language, size, nbPages //PrintTextValues textValues = DownloadPrint.PrintTextValuesManager.GetTextValues(xe.DescendantTextList(nodeFilter: node => !(node is XElement) || ((XElement)node).Name != "a"), data.Title); //data.Description = textValues.description; //data.Language = textValues.language; //data.Size = textValues.size; //data.NbPages = textValues.nbPages; //data.Infos.SetValues(textValues.infos); //data.DownloadLinks = xe.XPathValues(".//a/@href"); if (__trace) { pb.Trace.WriteLine(data.zToJson()); } return(data); }