private IEnumerator <FreeTelechargement_PostHeader> _Load(string url, bool reload, bool loadImage)
{
if (url != null)
{
// dont use mongo to store header page so key is null and refreshDocumentStore is false
RequestWebData_v3 request = new RequestWebData_v3(new RequestFromWeb_v3(url, _requestParameters, reload, loadImage));
_headerPage = FreeTelechargement_LoadHeaderPage.Load.Load(request).Document;
if (loadImage)
{
foreach (var postHeader in _headerPage.postHeaders)
{
FreeTelechargement.LoadImages(postHeader.images);
}
}
if (_headerPage != null)
{
return(_headerPage.postHeaders.AsEnumerable <FreeTelechargement_PostHeader>().GetEnumerator());
}
}
return(null);
}
protected override FreeTelechargement_PostDetail GetDataFromWeb(LoadDataFromWeb_v3 loadDataFromWeb)
{
throw new PBException("attention mismatch between free-telechargement.org and golden-ddl.net");
#pragma warning disable 162
XXElement xeSource = new XXElement(loadDataFromWeb.GetXmlDocument().Root);
FreeTelechargement_PostDetail data = new FreeTelechargement_PostDetail();
data.sourceUrl = loadDataFromWeb.request.Url;
data.loadFromWebDate = loadDataFromWeb.loadFromWebDate;
data.id = GetPostDetailKey(data.sourceUrl);
XXElement xePost = xeSource.XPathElement("//div[@id='dle-content']");
//data.category = xePost.DescendantTextList(".//div[@class='hdiin']//a").Select(FreeTelechargement.TrimFunc1).zToStringValues("/");
data.category = xePost.XPathElements(".//div[@class='hdiin']//a").DescendantTexts().Select(DownloadPrint.Trim).zToStringValues("/");
string category = data.category.ToLowerInvariant();
data.printType = GetPrintType(category);
//data.title = xePost.XPathValue(".//div[@class='bheading']//text()", FreeTelechargement.TrimFunc1);
data.title = xePost.XPathValue(".//div[@class='bheading']//text()").Trim(DownloadPrint.TrimChars);
PrintTitleInfos titleInfos = FreeTelechargement.PrintTextValuesManager.ExtractTitleInfos(data.title);
if (titleInfos.foundInfo)
{
data.originalTitle = data.title;
data.title = titleInfos.title;
data.infos.SetValues(titleInfos.infos);
}
string date = xePost.XPathValue(".//div[@class='datenews']//text()");
data.creationDate = zdate.ParseDateTimeLikeToday(date, loadDataFromWeb.loadFromWebDate, "d-M-yyyy, HH:mm");
if (data.creationDate == null)
{
pb.Trace.WriteLine("unknow date time \"{0}\"", date);
}
if (__trace)
{
pb.Trace.WriteLine("creationDate {0} - \"{1}\"", data.creationDate, date);
}
data.postAuthor = xePost.XPathValue(".//div[@class='argr']//a//text()");
XXElement xe = xePost.XPathElement(".//div[@class='maincont']");
//data.images = xe.XPathImages(xeImg => new UrlImage(zurl.GetUrl(loadDataFromWeb.request.Url, xeImg.zAttribValue("src")))).ToList();
data.images = xe.DescendantNodes(node => XmlDescendant.ImageFilter(node)).Select(xeImg => new WebImage(zurl.GetUrl(loadDataFromWeb.request.Url, xeImg.zAttribValue("src")))).ToList();
// force load image to get image width and height
if (loadDataFromWeb.request.LoadImage)
{
FreeTelechargement.LoadImages(data.images);
}
// get infos, description, language, size, nbPages
//PrintTextValues_old textValues = FreeTelechargement.PrintTextValuesManager.GetTextValues_old(xe.DescendantTextList(nodeFilter: node => !(node is XElement) || ((XElement)node).Name != "a"), data.title);
PrintTextValues_v1 textValues = FreeTelechargement.PrintTextValuesManager.GetTextValues_v1(xe.DescendantTexts(node => !(node is XElement) || ((XElement)node).Name != "a" ? XNodeFilter.SelectNode : XNodeFilter.SkipNode), data.title);
data.description = textValues.description;
data.language = textValues.language;
data.size = textValues.size;
data.nbPages = textValues.nbPages;
data.infos.SetValues(textValues.infos);
foreach (XXElement xe2 in xe.XPathElements("div/div"))
{
// http://prezup.eu http://pixhst.com/avaxhome/27/36/002e3627.jpeg http://www.zupmage.eu/i/R1UgqdXn4F.jpg
// http://i.imgur.com/Gu7hagN.jpg http://img11.hostingpics.net/pics/591623liens.png http://www.hapshack.com/images/jUfTZ.gif
// http://pixhst.com/pictures/3029467
data.downloadLinks.AddRange(xe2.XPathValues(".//a/@href").Where(url => !url.StartsWith("http://prezup.eu") && !url.StartsWith("http://pixhst.com") &&
!url.EndsWith(".jpg") && !url.EndsWith("jpeg") && !url.EndsWith("png") && !url.EndsWith("gif")));
}
if (__trace)
{
pb.Trace.WriteLine(data.zToJson());
}
return(data);
#pragma warning restore 162
}
