public void Load()
{
_loadUrlResult = false;
string url = _url;
if (_urlFile != null)
{
if (_reload || !zFile.Exists(_urlFile))
{
//if (!Http2.LoadToFile(_url, _urlFile))
if (!Http_v3.LoadToFile(_url, _urlFile))
{
return;
}
}
url = _urlFile;
}
//if (Http2.LoadUrl(url))
if (Http_v3.LoadUrl(url))
{
//SetXml(Http2.HtmlReader.XDocument.Root);
SetXml(Http_v3.Http.zGetXDocument().Root);
_loadUrlResult = true;
return;
}
}
protected void LoadUrl()
{
_loadUrlResult = false;
string url = _url;
if (_urlFile != null)
{
if (_reload || !zFile.Exists(_urlFile))
{
//if (!Http2.LoadToFile(_url, _urlFile))
if (!Http_v3.LoadToFile(_url, _urlFile))
{
return;
}
}
url = _urlFile;
}
//if (Http2.LoadUrl(url))
if (Http_v3.LoadUrl(url))
{
//**************************************************************************
// new from LoadFromWebBase1
//_xelement = new XXElement(Http2.HtmlReader.XDocument.Root);
_xelement = new XXElement(Http_v3.Http.zGetXDocument().Root);
_xmlEnum = GetXmlEnumerator();
//**************************************************************************
_loadUrlResult = true;
return;
}
}
public void Load(pb.Web.v1.RequestFromWeb_v2 request)
{
string url = request.Url;
if (_urlCache != null)
{
string urlPath = _urlCache.GetUrlPath(request.Url, request.RequestParameters);
if (request.ReloadFromWeb || !zFile.Exists(urlPath))
{
//if (!Http2.LoadToFile(request.Url, urlPath, request.RequestParameters))
// return;
if (!Http_v3.LoadToFile(request.Url, urlPath, request.RequestParameters))
{
return;
}
}
url = urlPath;
}
//if (!Http2.LoadUrl(url, request.RequestParameters))
// return;
if (!Http_v3.LoadUrl(url, request.RequestParameters))
{
return;
}
request.LoadResult = true;
}
public static void Test_Unea_request_01()
{
// req region :
// method post
// url http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp
// Cookie __atuvc=1%7C4; PHPSESSID=t85siq9oqn4sqmi0eg09schbh0; ASPSESSIONIDQABATCST=BDBNKJFAHJNFHLOCOJGCGOOF; __utma=169855717.692259951.1390462116.1391519633.1391522222.4; __utmc=169855717; __utmz=169855717.1390462116.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)
// Referer http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp?txtRecherche=Entrez%20le%20nom%20d'une%20entreprise&hiddenValider=true
// postData hiddenValider=true&txtRecherche=Par+Nom+Entreprise&txtRecherche1=&txtRecherche2=1&txtRecherche3=&txtRecherche4=
// txtRecherche2=1 ==> 1 = Alsace
string url = "http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp";
string request = "hiddenValider=true&txtRecherche=Par+Nom+Entreprise&txtRecherche1=&txtRecherche2=1&txtRecherche3=&txtRecherche4=";
string referer = "http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp?txtRecherche=Entrez%20le%20nom%20d'une%20entreprise&hiddenValider=true";
//HtmlXmlReader hr = Http2.HtmlReader;
//hr.WebRequestMethod = HttpRequestMethod.Post;
//hr.WebRequestContent = request;
//hr.WebRequestContentType = "application/x-www-form-urlencoded";
//hr.WebRequestReferer = referer;
HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();
requestParameters.method = HttpRequestMethod.Post;
requestParameters.content = request;
requestParameters.contentType = "application/x-www-form-urlencoded";
requestParameters.referer = referer;
Http_v3.LoadUrl(url, requestParameters);
}
public T Load(RequestFromWeb_v2 request)
{
string url = request.Url;
if (_urlCache != null)
{
string urlPath = _urlCache.GetUrlPath(request.Url, request.RequestParameters);
if (request.ReloadFromWeb || !zFile.Exists(urlPath))
{
//if (!Http2.LoadToFile(request.Url, urlPath, request.RequestParameters))
if (!Http_v3.LoadToFile(request.Url, urlPath, request.RequestParameters))
{
return(default(T));
}
}
url = urlPath;
}
//if (!Http2.LoadUrl(url, request.RequestParameters))
if (!Http_v3.LoadUrl(url, request.RequestParameters))
{
return(default(T));
}
request.LoadResult = true;
return(_getDataFromWeb(request));
}
public bool Load()
{
_loadResult = false;
string url = _url;
if (_urlCache != null)
{
//string urlPath = _urlCache.UrlPath;
string urlPath = _urlCache.GetUrlPath(url, _requestParameters);
if (_reload || !zFile.Exists(urlPath))
{
//if (!Http2.LoadToFile(_url, urlPath, _requestParameters))
if (!Http_v3.LoadToFile(_url, urlPath, _requestParameters))
{
return(false);
}
}
url = urlPath;
}
//if (!Http2.LoadUrl(url, _requestParameters))
if (!Http_v3.LoadUrl(url, _requestParameters))
{
return(false);
}
_loadResult = true;
_data = GetData();
return(true);
}
public static void Test_Handeco_Load_CompanyDetail_01()
{
string url = "http://www.handeco.org/fournisseur/consulter/id/1906";
Http_v3.LoadUrl(url, new HttpRequestParameters_v1 {
encoding = Encoding.UTF8
});
//HtmlXmlReader.CurrentHtmlXmlReader.http.Close();
//HtmlXmlReader.CurrentHtmlXmlReader.http.Dispose();
}
public T Load(RequestFromWeb_v3 request)
{
LoadDataFromWeb_v3 loadDataFromWeb = new LoadDataFromWeb_v3 {
request = request
};
DateTime loadFromWebDate;
string url = request.Url;
if (_urlCache != null)
{
string urlPath = _urlCache.GetUrlPath(url, request.RequestParameters);
if (request.ReloadFromWeb || !zFile.Exists(urlPath))
{
if (_firstLoadFromWeb && url.StartsWith("http://"))
{
InitLoadFromWeb();
_firstLoadFromWeb = false;
}
//if (!Http2.LoadToFile(url, urlPath, request.RequestParameters))
// return default(T);
if (!Http_v3.LoadToFile(url, urlPath, request.RequestParameters))
{
return(default(T));
}
}
url = urlPath;
// get last write time as loadFromWebDate, dont take creation time because creation time is modified when copying the file
//loadFromWebDate = new FileInfo(urlPath).LastWriteTime;
loadFromWebDate = zFile.CreateFileInfo(urlPath).LastWriteTime;
}
else
{
loadFromWebDate = DateTime.Now;
}
if (_firstLoadFromWeb && url.StartsWith("http://"))
{
InitLoadFromWeb();
_firstLoadFromWeb = false;
}
//if (!Http2.LoadUrl(url, request.RequestParameters))
// return default(T);
if (!Http_v3.LoadUrl(url, request.RequestParameters))
{
return(default(T));
}
loadDataFromWeb.loadResult = true;
loadDataFromWeb.loadFromWebDate = loadFromWebDate;
return(GetDataFromWeb(loadDataFromWeb));
}
protected MongoImage CreateMongoImage(string url, HttpRequestParameters_v1 requestParameters = null)
{
string file = _urlCache.GetUrlSubPath(url, requestParameters);
string path = zPath.Combine(_urlCache.CacheDirectory, file);
if (!zFile.Exists(path))
{
//Http2.LoadToFile(url, path, requestParameters);
Http_v3.LoadToFile(url, path, requestParameters);
}
Image image = null;
if (zFile.Exists(path))
{
try
{
image = zimg.LoadFromFile(path);
}
catch (Exception exception)
{
Trace.WriteLine("error unable to load image url \"{0}\" to \"{1}\" (WebImageMongoCacheManager.CreateMongoImage())", url, path);
Trace.Write("error : ");
Trace.WriteLine(exception.Message);
}
}
else
{
Trace.WriteLine("error unable to load image url \"{0}\" to \"{1}\" (WebImageMongoCacheManager.CreateMongoImage())", url, path);
}
MongoImage mongoImage = new MongoImage();
mongoImage.Url = url;
mongoImage.File = file;
mongoImage.Width = image != null ? image.Width : 0;
mongoImage.Height = image != null ? image.Height : 0;
mongoImage.Image = image;
GetCollection().zUpdate(new QueryDocument {
{ "_id", BsonValue.Create(url) }
}, new UpdateDocument {
{ "$set", mongoImage.ToBsonDocument() }
}, UpdateFlags.Upsert);
return(mongoImage);
}
public static void Test_Handeco_LoadHeaderPages_01(int nbPage = 1, bool loadLastPage = false)
{
Trace.WriteLine("Test_Handeco_Load_01");
// www.handeco.org => 87.98.177.160
// impossible d'aller directement à la page 2
// les paramètres sont envoyés par un post qui renvoie la page 1
// puis on fait un get pour les pages suivantes
string url = "http://www.handeco.org/fournisseurs/rechercher";
//Http2.LoadUrl(url);
// tous les départements
string content = "raisonSociale=&SIRET=&departements%5B%5D=67&departements%5B%5D=68&departements%5B%5D=24&departements%5B%5D=33&departements%5B%5D=40&departements%5B%5D=47&departements%5B%5D=64&departements%5B%5D=03&departements%5B%5D=15&departements%5B%5D=43&departements%5B%5D=63&departements%5B%5D=14&departements%5B%5D=50&departements%5B%5D=61&departements%5B%5D=21&departements%5B%5D=58&departements%5B%5D=71&departements%5B%5D=89&departements%5B%5D=22&departements%5B%5D=29&departements%5B%5D=35&departements%5B%5D=56&departements%5B%5D=18&departements%5B%5D=28&departements%5B%5D=36&departements%5B%5D=37&departements%5B%5D=41&departements%5B%5D=45&departements%5B%5D=08&departements%5B%5D=10&departements%5B%5D=51&departements%5B%5D=52&departements%5B%5D=2A&departements%5B%5D=2B&departements%5B%5D=25&departements%5B%5D=39&departements%5B%5D=70&departements%5B%5D=90&departements%5B%5D=27&departements%5B%5D=76&departements%5B%5D=75&departements%5B%5D=77&departements%5B%5D=78&departements%5B%5D=91&departements%5B%5D=92&departements%5B%5D=93&departements%5B%5D=94&departements%5B%5D=95&departements%5B%5D=11&departements%5B%5D=30&departements%5B%5D=34&departements%5B%5D=48&departements%5B%5D=66&departements%5B%5D=19&departements%5B%5D=23&departements%5B%5D=87&departements%5B%5D=54&departements%5B%5D=55&departements%5B%5D=57&departements%5B%5D=88&departements%5B%5D=09&departements%5B%5D=12&departements%5B%5D=31&departements%5B%5D=32&departements%5B%5D=46&departements%5B%5D=65&departements%5B%5D=81&departements%5B%5D=82&departements%5B%5D=59&departements%5B%5D=62&departements%5B%5D=44&departements%5B%5D=49&departements%5B%5D=53&departements%5B%5D=72&departements%5B%5D=85&departements%5B%5D=02&departements%5B%5D=60&departements%5B%5D=80&departements%5B%5D=16&departements%5B%5D=17&departements%5B%5D=79&departements%5B%5D=86&departements%5B%5D=04&departements%5B%5D=05&departements%5B%5D=06&departements%5B%5D=13&departements%5B%5D=83&departements%5B%5D=84&departements%5B%5D=01&departements%5B%5D=07&departements%5B%5D=26&departements%5B%5D=38&departements%5B%5D=42&departements%5B%5D=69&departements%5B%5D=73&departements%5B%5D=74&departements%5B%5D=971&departements%5B%5D=973&departements%5B%5D=972&departements%5B%5D=974&departements%5B%5D=988&departements%5B%5D=987&departements%5B%5D=975&departements%5B%5D=976&departements%5B%5D=986&experience_cotraitance=0&motsCles=&submitRecherche=Rechercher";
// Auvergne - 03 Allier
//string content = "raisonSociale=&SIRET=&departements%5B%5D=03&experience_cotraitance=0&motsCles=&submitRecherche=Rechercher";
CookieContainer cookies = new CookieContainer();
HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();
requestParameters.cookies = cookies;
requestParameters.method = HttpRequestMethod.Post;
requestParameters.content = content;
//requestParameters.Expect100Continue = true;
//HtmlXmlReader hxr = HtmlXmlReader.CurrentHtmlXmlReader;
//hxr.LoadRepeatIfError = 1;
//hxr.Load(url, requestParameters);
Http_v3.LoadUrl(url, requestParameters);
requestParameters = new HttpRequestParameters_v1();
requestParameters.cookies = cookies;
int page = 1;
for (int i = 1; i < nbPage; i++)
{
// http://www.handeco.org/fournisseurs/rechercher/page/2
url = "http://www.handeco.org/fournisseurs/rechercher/page/" + (++page).ToString();
Http_v3.LoadUrl(url, requestParameters);
}
if (loadLastPage)
{
url = "http://www.handeco.org/fournisseurs/rechercher/page/200";
Http_v3.LoadUrl(url, requestParameters);
}
}
public static void Test_01(string file, bool executeRequest = false)
{
Trace.WriteLine("file \"{0}\"", file);
HttpArchive har = new HttpArchive(file);
Trace.WriteLine("select first entry");
var entry = har.Entries.First();
string url = entry.request.GetHttpRequestUrl();
Trace.WriteLine("url \"{0}\"", url);
HttpRequestParameters_v1 requestParameters = entry.request.GetHttpRequestParameters();
Trace.WriteLine("requestParameters :");
Trace.WriteLine(requestParameters.zToJson());
if (executeRequest)
{
//Http2.LoadUrl(url, requestParameters);
Http_v3.LoadUrl(url, requestParameters);
}
}