public Handeco_LoadHeader(string url, HttpRequestParameters_v1 requestParameters = null)
: base(url, requestParameters)
{
//_imageCacheDirectory = __imageCacheDirectory;
SetXmlParameters(__useXml);
SetMongoParameters(__useMongo, __mongoServer, __mongoDatabase, __mongoCollectionName);
}
public RequestFromWeb_v3(string url, HttpRequestParameters_v1 requestParameters = null, bool reload = false, bool loadImage = false)
{
_url = url;
_requestParameters = requestParameters;
_reloadFromWeb = reload;
_loadImage = loadImage;
}
public static void Test_Unea_request_01()
{
// req region :
// method post
// url http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp
// Cookie __atuvc=1%7C4; PHPSESSID=t85siq9oqn4sqmi0eg09schbh0; ASPSESSIONIDQABATCST=BDBNKJFAHJNFHLOCOJGCGOOF; __utma=169855717.692259951.1390462116.1391519633.1391522222.4; __utmc=169855717; __utmz=169855717.1390462116.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)
// Referer http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp?txtRecherche=Entrez%20le%20nom%20d'une%20entreprise&hiddenValider=true
// postData hiddenValider=true&txtRecherche=Par+Nom+Entreprise&txtRecherche1=&txtRecherche2=1&txtRecherche3=&txtRecherche4=
// txtRecherche2=1 ==> 1 = Alsace
string url = "http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp";
string request = "hiddenValider=true&txtRecherche=Par+Nom+Entreprise&txtRecherche1=&txtRecherche2=1&txtRecherche3=&txtRecherche4=";
string referer = "http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp?txtRecherche=Entrez%20le%20nom%20d'une%20entreprise&hiddenValider=true";
//HtmlXmlReader hr = Http2.HtmlReader;
//hr.WebRequestMethod = HttpRequestMethod.Post;
//hr.WebRequestContent = request;
//hr.WebRequestContentType = "application/x-www-form-urlencoded";
//hr.WebRequestReferer = referer;
HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();
requestParameters.method = HttpRequestMethod.Post;
requestParameters.content = request;
requestParameters.contentType = "application/x-www-form-urlencoded";
requestParameters.referer = referer;
Http_v3.LoadUrl(url, requestParameters);
}
public IEnumerable <TData2> LoadPages(int startPage = 1, int maxPage = 1, bool reload = false, bool loadImage = false, bool refreshDocumentStore = false)
{
HttpRequestParameters_v1 httpRequestParameters = GetHttpRequestParameters();
string url = GetUrlPage(startPage);
IEnumDataPages_v1 <TKey, TData2> dataPage = Load(new RequestWebData_v4 <TKey>(new RequestFromWeb_v3(url, httpRequestParameters, reload, loadImage), GetKeyFromUrl(url), refreshDocumentStore)).Document;
if (dataPage == null)
{
yield break;
}
foreach (TData2 data in dataPage.GetDataList())
{
yield return(data);
}
for (int nbPage = 1; nbPage < maxPage; nbPage++)
{
url = dataPage.GetUrlNextPage();
if (url == null)
{
break;
}
dataPage = Load(new RequestWebData_v4 <TKey>(new RequestFromWeb_v3(url, httpRequestParameters, reload, loadImage), GetKeyFromUrl(url), refreshDocumentStore)).Document;
foreach (TData2 data in dataPage.GetDataList())
{
yield return(data);
}
}
}
protected override HttpRequestParameters_v1 GetHttpRequestParameters()
{
HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();
requestParameters.encoding = Encoding.UTF8;
return(requestParameters);
}
//public static CookieContainer Cookies { get { return _hxr.Cookies; } }
public static bool LoadUrl(string url, HttpRequestParameters_v1 requestParameters = null)
{
try
{
_hxr.Load(url, requestParameters);
//WriteLine("request headers :");
//WriteHeaders(_hxr.http.Request.Headers);
//WriteLine("response headers :");
//WriteHeaders(_hxr.http.Response.Headers);
return(true);
}
catch (Exception ex)
{
//Load("http://www.frboard.com/magazines-et-journaux/441873-multi-les-journaux-mardi-13-aout-2013-pdf-lien-direct.html");
//15/08/2013 12:00:32 Error : A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond 5.199.168.178:80 (System.Net.Sockets.SocketException)
//Unable to connect to the remote server (System.Net.WebException)
//----------------------
// at System.Net.Sockets.Socket.DoConnect(EndPoint endPointSnapshot, SocketAddress socketAddress)
// at System.Net.ServicePoint.ConnectSocketInternal(Boolean connectFailure, Socket s4, Socket s6, Socket& socket, IPAddress& address, ConnectSocketState state, IAsyncResult asyncResult, Int32 timeout, Exception& exception)
//----------------------
// at System.Net.HttpWebRequest.GetResponse()
// at pb.old.Http.OpenWebRequest() in c:\pib\dropbox\pbeuz\Dropbox\dev\project\Source\Source_01\Source\PB_Tools\\Http_Html.cs:line 911
// at pb.old.Http.Open() in c:\pib\dropbox\pbeuz\Dropbox\dev\project\Source\Source_01\Source\PB_Tools\\Http_Html.cs:line 780
// at pb.old.Http.Load() in c:\pib\dropbox\pbeuz\Dropbox\dev\project\Source\Source_01\Source\PB_Tools\\Http_Html.cs:line 503
// at pb.old.HtmlXmlReader.Load(String sUrl) in c:\pib\dropbox\pbeuz\Dropbox\dev\project\Source\Source_01\Source\PB_Tools\\HtmlXmlReader.cs:line 426
// at Print.download.w.Test_frboard_02()
// at Print.download.w.Run()
//Trace.WriteLine("Error : \"{0}\" ({1})", ex.Message, ex.GetType().ToString());
Trace.CurrentTrace.WriteError(ex);
return(false);
}
}
protected override HttpRequestParameters_v1 GetHttpRequestParameters()
{
HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();
requestParameters.encoding = Encoding.UTF8;
requestParameters.cookies.Add(new Uri(__url), new Cookie("hasVisitedSite", "Yes"));
return(requestParameters);
}
public static TelechargementPlus_PostDetail LoadPostDetailFromWeb(string url, bool reload = false, bool loadImage = false)
{
HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();
requestParameters.encoding = Encoding.UTF8;
pb.Web.v1.RequestFromWeb_v2 request = new pb.Web.v1.RequestFromWeb_v2(url, requestParameters, reload, loadImage);
return(new pb.Web.v1.LoadDataFromWeb_v2 <TelechargementPlus_PostDetail>(TelechargementPlus_LoadDetail.LoadPostDetailFromWeb, TelechargementPlus_LoadDetail.GetUrlCache()).Load(request));
}
public static void Test_RapideDdl_LoadUrl_01(string url)
{
//string url = "http://rapide-ddl.com/ebooks/";
HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();
requestParameters.encoding = Encoding.UTF8;
requestParameters.cookies.Add(new Uri(url), new Cookie("hasVisitedSite", "Yes"));
pb.old.Http_v2.LoadUrl(url, requestParameters);
}
public override Image LoadImage(string url, HttpRequestParameters_v1 requestParameters = null)
{
MongoImage mongoImage = LoadMongoImage(url, requestParameters);
if (mongoImage.Image == null)
{
LoadImage(mongoImage);
}
return(mongoImage.Image);
}
public LoadWebData_v4 <TKey, TData> LoadDocument(string url, HttpRequestParameters_v1 requestParameters = null, bool reloadFromWeb = false, bool loadImage = false, bool refreshDocumentStore = false)
{
if (requestParameters == null)
{
requestParameters = GetHttpRequestParameters();
}
LoadWebData_v4 <TKey, TData> loadWebData = _loadWebDataManager.Load(new RequestWebData_v4 <TKey>(new RequestFromWeb_v3(url, requestParameters, reloadFromWeb, loadImage), GetKeyFromUrl(url), refreshDocumentStore));
return(loadWebData);
}
//protected override string GetUrlPage(int page)
protected override void GetUrlPage(int page, out string url, out HttpRequestParameters_v1 requestParameters)
{
//http://www.reseau-gesat.com/Gesat/EtablissementList-10-10.html
url = _urlHeader;
if (page != 1)
{
url += string.Format("EtablissementList-{0}-10.html", (page - 1) * _headersNumberByPage);
}
requestParameters = null;
}
public Handeco_LoadHeaderFromWeb(string url, HttpRequestParameters_v1 requestParameters = null, bool reload = false, bool loadImage = false)
: base(url, requestParameters, reload)
{
//SetRequestParameters(new HttpRequestParameters() { encoding = Encoding.UTF8 });
if (__useUrlCache)
{
SetUrlCache(new UrlCache_v1(__cacheDirectory, __urlFileNameType));
}
_loadImage = loadImage;
}
protected override bool GetUrlNextPage(out string url, out HttpRequestParameters_v1 requestParameters)
{
url = null;
requestParameters = null;
if (++_page > __regionCodes.Length)
{
return(false);
}
GetUrlPage(_page, out url, out requestParameters);
return(true);
}
public string GetUrlSubPath(string url, HttpRequestParameters_v1 requestParameters = null)
{
string file = GetUrlFilename(url, requestParameters);
string dir = GetUrlSubDirectory(url, requestParameters);
if (dir != null)
{
file = zPath.Combine(dir, file);
}
return(file);
}
public virtual string GetUrlSubDirectory(string url, HttpRequestParameters_v1 requestParameters = null)
{
if (_getUrlSubDirectory != null)
{
return(_getUrlSubDirectory(url, requestParameters));
}
else
{
return(null);
}
}
protected override IEnumerator <FreeTelechargement_PostHeader> LoadPage(int page, bool reload, bool loadImage)
{
// http://www.free-telechargement.org/2/categorie-Magazines/
if (page < 1)
{
throw new PBException("error wrong page number {0}", page);
}
string url = string.Format(__url, page);
_requestParameters = new HttpRequestParameters_v1();
_requestParameters.encoding = Encoding.Default;
return(_Load(url, reload, loadImage));
}
//protected override string GetUrlNextPage()
protected override bool GetUrlNextPage(out string url, out HttpRequestParameters_v1 requestParameters)
{
url = _headerPage.urlNextPage;
requestParameters = null;
if (url != null)
{
return(true);
}
else
{
return(false);
}
}
public static void Test_UrlToFileName_01()
{
string url = "http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp";
string request = "hiddenValider=true&txtRecherche=&txtRecherche1=&txtRecherche2=1&txtRecherche3=&txtRecherche4=";
HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();
requestParameters.method = HttpRequestMethod.Post;
requestParameters.content = request;
//_tr.WriteLine(zurl.UrlToFileName(url, UrlFileNameType.FileName | UrlFileNameType.Content, null, requestParameters));
_tr.WriteLine(zurl.UrlToFileName(new HttpRequest {
Url = url, Content = requestParameters.content
}, UrlFileNameType.FileName | UrlFileNameType.Content));
}
public static void Test_LoadWithCookies_01(string url)
{
HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();
string cookiesFile = XmlConfig.CurrentConfig.GetExplicit("Ebookdz/CookiesFile");
requestParameters.cookies = zcookies.LoadCookies(cookiesFile);
Trace.WriteLine("load cookies from \"{0}\"", cookiesFile);
pb.old.Http_v2.LoadUrl(url, requestParameters);
XXElement xeSource = new XXElement(pb.old.Http_v2.HtmlReader.XDocument.Root);
Trace.WriteLine("Login : \"{0}\"", Test_GetLogin_01(xeSource));
Trace.WriteLine("Is logged in : \"{0}\"", Test_IsLoggedIn_01(xeSource));
}
public MongoImage LoadMongoImage(string url, HttpRequestParameters_v1 requestParameters = null)
{
BsonDocument document = GetCollection().zFindOneById <BsonDocument>(BsonValue.Create(url));
if (document != null)
{
return(BsonSerializer.Deserialize <MongoImage>(document));
}
else
{
return(CreateMongoImage(url, requestParameters));
}
}
public static bool LoadToFile(string url, string file, HttpRequestParameters_v1 requestParameters = null)
{
try
{
_hxr.LoadToFile(url, file, requestParameters);
return(true);
}
catch (Exception ex)
{
//WriteLine("Error : \"{0}\" ({1})", ex.Message, ex.GetType().ToString());
Trace.CurrentTrace.WriteError(ex);
return(false);
}
}
protected override void GetUrlPage(int page, out string url, out HttpRequestParameters_v1 requestParameters)
{
if (page != 1)
{
throw new PB_Util_Exception("error impossible to load directly page n, only page 1 must be loaded first");
}
_page = page;
url = __url;
requestParameters = new HttpRequestParameters_v1();
requestParameters.method = HttpRequestMethod.Post;
requestParameters.content = __content;
requestParameters.encoding = Encoding.UTF8;
//_requestParameters = requestParameters;
}
public static void LoadImages(List <WebImage> images, HttpRequestParameters_v1 requestParameters = null)
{
for (int i = 0; i < images.Count;)
{
if (!LoadImage(images[i], requestParameters, __imageFilter))
{
images.RemoveAt(i);
}
else
{
i++;
}
}
}
protected override void GetUrlPage(int page, out string url, out HttpRequestParameters_v1 requestParameters)
{
if (page > __regionCodes.Length)
{
throw new PB_Util_Exception("error loading Unea header page {0} dont exist", page);
}
_page = page;
url = __url;
requestParameters = new HttpRequestParameters_v1();
requestParameters.method = HttpRequestMethod.Post;
requestParameters.content = string.Format(__regionRequest, __regionCodes[page - 1]);
//requestParameters.contentType = __requestContentType;
requestParameters.referer = __referer;
}
public static RapideDdl_HeaderPage Load(string url, HttpRequestParameters_v1 requestParameters = null, bool reload = false, bool loadImage = false)
{
pb.Web.v1.RequestFromWeb_v2 request = new pb.Web.v1.RequestFromWeb_v2(url, requestParameters, reload, loadImage);
//return _load.Load(request);
RapideDdl_HeaderPage headerPage = _load.Load(request);
if (loadImage)
{
foreach (var postHeader in headerPage.postHeaders)
{
pb.old.Http_v2.LoadImageFromWeb(postHeader.images);
}
}
return(headerPage);
}
//public Unea_LoadHeaderFromWeb(string request, bool reload = false, bool loadImage = false)
public Unea_LoadHeaderFromWeb(string url, HttpRequestParameters_v1 requestParameters, bool reload = false, bool loadImage = false)
: base(url, requestParameters, reload)
{
//HttpRequestParameters requestParameters = new HttpRequestParameters();
//requestParameters.method = HttpRequestMethod.Post;
//requestParameters.content = request;
//requestParameters.contentType = __requestContentType;
//requestParameters.referer = __referer;
//SetRequestParameters(requestParameters);
if (__useUrlCache)
{
SetUrlCache(new UrlCache_v1(__cacheDirectory, __urlFileNameType));
}
_loadImage = loadImage;
}
protected override void GetUrlPage(int page, out string url, out HttpRequestParameters_v1 requestParameters)
{
// http://www.telechargement-plus.com/e-book-magazines/page/2/
_page = page;
url = __url;
if (page < 1)
{
throw new PBException("error wrong page number {0}", page);
}
if (page > 1)
{
url += string.Format("page/{0}/", page);
}
requestParameters = new HttpRequestParameters_v1();
requestParameters.encoding = Encoding.UTF8;
}
public static Image LoadImageFromWeb(string url, HttpRequestParameters_v1 requestParameters = null)
{
try
{
Image image = _hxr.LoadImage(url, requestParameters);
//if (image.Height > 200)
// image = image.zResize(height: 200);
return(image);
}
catch (Exception ex)
{
//WriteLine("Error : \"{0}\" ({1})", ex.Message, ex.GetType().ToString());
Trace.CurrentTrace.WriteError(ex);
return(null);
}
}
protected MongoImage CreateMongoImage(string url, HttpRequestParameters_v1 requestParameters = null)
{
string file = _urlCache.GetUrlSubPath(url, requestParameters);
string path = zPath.Combine(_urlCache.CacheDirectory, file);
if (!zFile.Exists(path))
{
//Http2.LoadToFile(url, path, requestParameters);
Http_v3.LoadToFile(url, path, requestParameters);
}
Image image = null;
if (zFile.Exists(path))
{
try
{
image = zimg.LoadFromFile(path);
}
catch (Exception exception)
{
Trace.WriteLine("error unable to load image url \"{0}\" to \"{1}\" (WebImageMongoCacheManager.CreateMongoImage())", url, path);
Trace.Write("error : ");
Trace.WriteLine(exception.Message);
}
}
else
{
Trace.WriteLine("error unable to load image url \"{0}\" to \"{1}\" (WebImageMongoCacheManager.CreateMongoImage())", url, path);
}
MongoImage mongoImage = new MongoImage();
mongoImage.Url = url;
mongoImage.File = file;
mongoImage.Width = image != null ? image.Width : 0;
mongoImage.Height = image != null ? image.Height : 0;
mongoImage.Image = image;
GetCollection().zUpdate(new QueryDocument {
{ "_id", BsonValue.Create(url) }
}, new UpdateDocument {
{ "$set", mongoImage.ToBsonDocument() }
}, UpdateFlags.Upsert);
return(mongoImage);
}
