public static void Test_LoadMongoDetail_01()
{
Trace.WriteLine("Test_LoadMongoDetail_01");
Trace.WriteLine();
RapideDdl.InitMongoClassMap();
if (BsonClassMap.IsClassMapRegistered(typeof(RapideDdl_Base)))
{
BsonClassMap map = BsonClassMap.LookupClassMap(typeof(RapideDdl_Base));
Trace.WriteLine("change existing class map");
BsonMemberMap memberMap = map.GetMemberMap("infos");
memberMap.SetSerializationOptions(DictionarySerializationOptions.ArrayOfDocuments);
}
else
{
Trace.WriteLine("register class map");
BsonClassMap.RegisterClassMap <RapideDdl_Base>(cm =>
{
cm.AutoMap();
cm.GetMemberMap(c => c.infos).SetSerializationOptions(DictionarySerializationOptions.ArrayOfDocuments);
});
}
Trace.WriteLine();
string query = "{ _id: 35105 }";
MongoCursor <BsonDocument> cursor = MongoCommand.GetDatabase(null, "dl").GetCollection("RapideDdl_Detail").zFind <BsonDocument>(new QueryDocument(BsonSerializer.Deserialize <BsonDocument>(query)));
int i = 1;
foreach (BsonDocument document in cursor)
{
BsonDocument document2 = (BsonDocument)document["download"];
RapideDdl_PostDetail postDetail = BsonSerializer.Deserialize <RapideDdl_PostDetail>(document2);
//MongoDB.Bson.Serialization.IBsonSerializationOptions options;
//MongoDB.Bson.Serialization.BsonSerializer.Serialize()
//Serialization.Options
//SerializationOptions
//MongoDB.Bson.Serialization.Options.RepresentationSerializationOptions
//document2 = postDetail.ToBsonDocument(new DictionarySerializationOptions(DictionaryRepresentation.ArrayOfDocuments));
//DictionarySerializationOptions.
//BsonClassMap.RegisterClassMap<RapideDdl_PostDetail>(cm => { cm.MapProperty(c => c.SomeProperty); cm.MapProperty(c => c.AnotherProperty); });
//document2 = postDetail.ToBsonDocument();
Trace.WriteLine("document no {0}", i++);
//DocumentSerializationOptions options = new DocumentSerializationOptions();
//Trace.WriteLine(postDetail.ToJson(new DictionarySerializationOptions(DictionaryRepresentation.ArrayOfDocuments)));
//RapideDdl_PostDetail RapideDdl_Base
//BsonClassMap<RapideDdl_Base> map = BsonClassMap.LookupClassMap(typeof(RapideDdl_Base));
//Trace.WriteLine(document.zToJson());
Trace.WriteLine(postDetail.zToJson());
Trace.WriteLine();
}
}
public static RapideDdl_PostDetail LoadPostDetailFromWeb(pb.Web.v1.RequestFromWeb_v2 request)
{
XXElement xeSource = new XXElement(request.GetXmlDocument().Root);
RapideDdl_PostDetail data = new RapideDdl_PostDetail();
data.sourceUrl = request.Url;
data.loadFromWebDate = DateTime.Now;
XXElement xePost = xeSource.XPathElement("//div[@class='lcolomn mainside']");
//data.category = xePost.DescendantTextList(".//div[@class='spbar']//a").Select(Download.Print.RapideDdl.RapideDdl.TrimFunc1).Where(s => s != "Accueil" && s != "").zToStringValues("/");
data.category = xePost.XPathElements(".//div[@class='spbar']//a").DescendantTexts().Select(Download.Print.RapideDdl.RapideDdl.TrimFunc1).Where(s => s != "Accueil" && s != "").zToStringValues("/");
//data.title = RapideDdl.ExtractTextValues(data.infos, xePost.XPathValue(".//div[@class='base fullstory']//text()", RapideDdl.TrimFunc1));
//data.title = xePost.DescendantTextList(".//div[@class='spbar']", node => !(node is XElement) || ((XElement)node).Name != "a", RapideDdl.TrimFunc1).FirstOrDefault();
//data.title = xePost.XPathValue(".//div[@class='spbar']/text()", RapideDdl.TrimFunc1);
//data.title = xePost.DescendantTextList(".//div[@class='spbar']", func: Download.Print.RapideDdl.RapideDdl.TrimFunc1).LastOrDefault();
data.title = xePost.XPathElements(".//div[@class='spbar']").DescendantTexts().Select(Download.Print.RapideDdl.RapideDdl.TrimFunc1).LastOrDefault();
XXElement xe = xePost.XPathElement(".//div[@class='shdinfo']");
//////////////data.creationDate = Download.Print.RapideDdl.RapideDdl.ParseDateTime(xe.XPathValue(".//span[@class='date']//text()"), (DateTime)data.loadFromWebDate);
data.postAuthor = xe.XPathValue(".//span[@class='arg']//a//text()");
xe = xePost.XPathElement(".//div[@class='maincont']");
//data.images = xe.XPathImages(request.Url, nodeFilter: node => node is XElement && ((XElement)node).Name == "a");
//data.images = xe.XPathImages(request.Url);
//data.images = xe.XPathImages(xeImg => new ImageHtml(xeImg, request.Url)).ToList();
data.images = xe.DescendantNodes(node => XmlDescendant.ImageFilter(node)).Select(xeImg => new pb.old.ImageHtml((XElement)xeImg, request.Url)).ToList();
//if (request.LoadImage)
// force load image to get image width and height
pb.old.Http_v2.LoadImageFromWeb(data.images);
//data.SetTextValues(xe.DescendantTextList(".//span[@id='post-img']", node => node is XElement && ((XElement)node).Name == "a" ? false : true));
//data.SetTextValues(xe.DescendantTextList(".//div"));
data.SetTextValues(xe.XPathElements(".//div").DescendantTexts());
//data.downloadLinks.AddRange(xe.XPathValues(".//div[2]//a/@href"));
//foreach (XXElement xe2 in xe.XPathElements("div/div").Skip(1))
foreach (XXElement xe2 in xe.XPathElements("div/div"))
{
// http://prezup.eu http://pixhst.com/avaxhome/27/36/002e3627.jpeg http://www.zupmage.eu/i/R1UgqdXn4F.jpg
// http://i.imgur.com/Gu7hagN.jpg http://img11.hostingpics.net/pics/591623liens.png http://www.hapshack.com/images/jUfTZ.gif
// http://pixhst.com/pictures/3029467
//data.downloadLinks.AddRange(xe2.XPathValues(".//a/@href").Where(url => !url.StartsWith("http://prezup.eu") && !url.StartsWith("http://pixhst.com") && !url.StartsWith("http://www.zupmage.eu")));
data.downloadLinks.AddRange(xe2.XPathValues(".//a/@href").Where(url => !url.StartsWith("http://prezup.eu") && !url.StartsWith("http://pixhst.com") &&
!url.EndsWith(".jpg") && !url.EndsWith("jpeg") && !url.EndsWith("png") && !url.EndsWith("gif")));
}
return(data);
}
public static void Test_RapideDdl_LoadDetail_01()
{
//string url = "http://www.rapide-ddl.com/ebooks/magazine/36143-secrets-dhistoire-de-dgtours-en-france-no1-2014.html";
//string url = "http://www.rapide-ddl.com/ebooks/magazine/36142-le-point-no2184-du-24-au-30-juillet-2014-lien-direct.html";
string url = "http://www.rapide-ddl.com/ebooks/bandes-dessinee/36158-x-men-curse-of-the-mutants-sgrie-en-cours-19-tomes-comicmulti.html";
//string url = "http://www.rapide-ddl.com/ebooks/bandes-dessinee/36153-lgonard-sgrie-en-cours-43-tomes-2hs-bdmulti.html";
//string url = "http://www.rapide-ddl.com/ebooks/bandes-dessinee/36152-lgon-la-terreur-sgrie-finie-9-tomes-bdmulti.html";
//string url = "http://www.rapide-ddl.com/ebooks/bandes-dessinee/36151-les-aventures-de-bessy-sgrie-finie-22-tomes-bdmulti.html";
RapideDdl_PostDetail post = RapideDdl_LoadDetail.Load(url, reload: false, loadImage: true, refreshDocumentStore: true);
//RunSource.CurrentRunSource.View(post);
post.zView();
}
// bool desactivateDocumentStore = false
public static RapideDdl_PostDetail Load(string url, HttpRequestParameters_v1 requestParameters = null, bool reload = false, bool loadImage = false, bool refreshDocumentStore = false)
{
//Trace.WriteLine("RapideDdl_LoadDetail.Load \"{0}\"", url);
if (requestParameters == null)
{
requestParameters = new HttpRequestParameters_v1();
}
requestParameters.encoding = Encoding.UTF8;
pb.Web.v1.RequestFromWeb_v2 request = new pb.Web.v1.RequestFromWeb_v2(url, requestParameters, reload, false);
//_load.DesactivateDocumentStore = desactivateDocumentStore;
RapideDdl_PostDetail postDetail = _load.Load(request, GetPostDetailKey(url), refreshDocumentStore);
//_load.DesactivateDocumentStore = false;
if (loadImage)
{
pb.old.Http_v2.LoadImageFromWeb(postDetail.images);
}
return(postDetail);
}
public static RapideDdl_PostDetail LoadPostDetailFromWeb(pb.Web.v1.RequestFromWeb_v2 request)
{
XXElement xeSource = new XXElement(request.GetXmlDocument().Root);
RapideDdl_PostDetail data = new RapideDdl_PostDetail();
data.sourceUrl = request.Url;
data.loadFromWebDate = DateTime.Now;
XXElement xePost = xeSource.XPathElement("//div[@class='lcolomn mainside']");
//data.category = xePost.DescendantTextList(".//div[@class='spbar']//a").Select(Download.Print.RapideDdl.RapideDdl.TrimFunc1).Where(s => s != "Accueil" && s != "").zToStringValues("/");
data.category = xePost.XPathElements(".//div[@class='spbar']//a").DescendantTexts().Select(Download.Print.RapideDdl.RapideDdl.TrimFunc1).Where(s => s != "Accueil" && s != "").zToStringValues("/");
//data.title = RapideDdl.ExtractTextValues(data.infos, xePost.XPathValue(".//div[@class='base fullstory']//text()", RapideDdl.TrimFunc1));
//data.title = xePost.DescendantTextList(".//div[@class='spbar']", node => !(node is XElement) || ((XElement)node).Name != "a", RapideDdl.TrimFunc1).FirstOrDefault();
//data.title = xePost.XPathValue(".//div[@class='spbar']/text()", RapideDdl.TrimFunc1);
//data.title = xePost.DescendantTextList(".//div[@class='spbar']", func: Download.Print.RapideDdl.RapideDdl.TrimFunc1).LastOrDefault();
data.title = xePost.XPathElements(".//div[@class='spbar']").DescendantTexts().Select(Download.Print.RapideDdl.RapideDdl.TrimFunc1).LastOrDefault();
XXElement xe = xePost.XPathElement(".//div[@class='shdinfo']");
//////////////data.creationDate = Download.Print.RapideDdl.RapideDdl.ParseDateTime(xe.XPathValue(".//span[@class='date']//text()"), (DateTime)data.loadFromWebDate);
data.postAuthor = xe.XPathValue(".//span[@class='arg']//a//text()");
xe = xePost.XPathElement(".//div[@class='maincont']");
//data.images = xe.XPathImages(request.Url, nodeFilter: node => node is XElement && ((XElement)node).Name == "a");
//data.images = xe.XPathImages(request.Url);
//data.images = xe.XPathImages(xeImg => new ImageHtml(xeImg, request.Url)).ToList();
data.images = xe.DescendantNodes(node => XmlDescendant.ImageFilter(node)).Select(xeImg => new pb.old.ImageHtml((XElement)xeImg, request.Url)).ToList();
//if (request.LoadImage)
// force load image to get image width and height
pb.old.Http_v2.LoadImageFromWeb(data.images);
//data.SetTextValues(xe.DescendantTextList(".//span[@id='post-img']", node => node is XElement && ((XElement)node).Name == "a" ? false : true));
//data.SetTextValues(xe.DescendantTextList(".//div"));
data.SetTextValues(xe.XPathElements(".//div").DescendantTexts());
//data.downloadLinks.AddRange(xe.XPathValues(".//div[2]//a/@href"));
//foreach (XXElement xe2 in xe.XPathElements("div/div").Skip(1))
foreach (XXElement xe2 in xe.XPathElements("div/div"))
{
// http://prezup.eu http://pixhst.com/avaxhome/27/36/002e3627.jpeg http://www.zupmage.eu/i/R1UgqdXn4F.jpg
// http://i.imgur.com/Gu7hagN.jpg http://img11.hostingpics.net/pics/591623liens.png http://www.hapshack.com/images/jUfTZ.gif
// http://pixhst.com/pictures/3029467
//data.downloadLinks.AddRange(xe2.XPathValues(".//a/@href").Where(url => !url.StartsWith("http://prezup.eu") && !url.StartsWith("http://pixhst.com") && !url.StartsWith("http://www.zupmage.eu")));
data.downloadLinks.AddRange(xe2.XPathValues(".//a/@href").Where(url => !url.StartsWith("http://prezup.eu") && !url.StartsWith("http://pixhst.com")
&& !url.EndsWith(".jpg") && !url.EndsWith("jpeg") && !url.EndsWith("png") && !url.EndsWith("gif")));
}
return data;
}
