Exemple #1
0
        public void Load()
        {
            _loadUrlResult = false;
            string url = _url;

            if (_urlFile != null)
            {
                if (_reload || !zFile.Exists(_urlFile))
                {
                    //if (!Http2.LoadToFile(_url, _urlFile))
                    if (!Http_v3.LoadToFile(_url, _urlFile))
                    {
                        return;
                    }
                }
                url = _urlFile;
            }
            //if (Http2.LoadUrl(url))
            if (Http_v3.LoadUrl(url))
            {
                //SetXml(Http2.HtmlReader.XDocument.Root);
                SetXml(Http_v3.Http.zGetXDocument().Root);
                _loadUrlResult = true;
                return;
            }
        }
Exemple #2
0
        protected void LoadUrl()
        {
            _loadUrlResult = false;
            string url = _url;

            if (_urlFile != null)
            {
                if (_reload || !zFile.Exists(_urlFile))
                {
                    //if (!Http2.LoadToFile(_url, _urlFile))
                    if (!Http_v3.LoadToFile(_url, _urlFile))
                    {
                        return;
                    }
                }
                url = _urlFile;
            }
            //if (Http2.LoadUrl(url))
            if (Http_v3.LoadUrl(url))
            {
                //**************************************************************************
                // new from LoadFromWebBase1
                //_xelement = new XXElement(Http2.HtmlReader.XDocument.Root);
                _xelement = new XXElement(Http_v3.Http.zGetXDocument().Root);
                _xmlEnum  = GetXmlEnumerator();
                //**************************************************************************
                _loadUrlResult = true;
                return;
            }
        }
Exemple #3
0
        public void Load(pb.Web.v1.RequestFromWeb_v2 request)
        {
            string url = request.Url;

            if (_urlCache != null)
            {
                string urlPath = _urlCache.GetUrlPath(request.Url, request.RequestParameters);
                if (request.ReloadFromWeb || !zFile.Exists(urlPath))
                {
                    //if (!Http2.LoadToFile(request.Url, urlPath, request.RequestParameters))
                    //    return;
                    if (!Http_v3.LoadToFile(request.Url, urlPath, request.RequestParameters))
                    {
                        return;
                    }
                }
                url = urlPath;
            }
            //if (!Http2.LoadUrl(url, request.RequestParameters))
            //    return;
            if (!Http_v3.LoadUrl(url, request.RequestParameters))
            {
                return;
            }
            request.LoadResult = true;
        }
Exemple #4
0
        public static void Test_Unea_request_01()
        {
            //   req region :
            //     method post
            //     url http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp
            //     Cookie __atuvc=1%7C4; PHPSESSID=t85siq9oqn4sqmi0eg09schbh0; ASPSESSIONIDQABATCST=BDBNKJFAHJNFHLOCOJGCGOOF; __utma=169855717.692259951.1390462116.1391519633.1391522222.4; __utmc=169855717; __utmz=169855717.1390462116.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)
            //     Referer http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp?txtRecherche=Entrez%20le%20nom%20d'une%20entreprise&hiddenValider=true
            //     postData hiddenValider=true&txtRecherche=Par+Nom+Entreprise&txtRecherche1=&txtRecherche2=1&txtRecherche3=&txtRecherche4=
            //     txtRecherche2=1 ==> 1 = Alsace

            string url     = "http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp";
            string request = "hiddenValider=true&txtRecherche=Par+Nom+Entreprise&txtRecherche1=&txtRecherche2=1&txtRecherche3=&txtRecherche4=";
            string referer = "http://www.unea.fr/union-nationale-entreprises-adaptees/annuaire-unea/71/71/annuaire/annuaire.asp?txtRecherche=Entrez%20le%20nom%20d'une%20entreprise&hiddenValider=true";

            //HtmlXmlReader hr = Http2.HtmlReader;
            //hr.WebRequestMethod = HttpRequestMethod.Post;
            //hr.WebRequestContent = request;
            //hr.WebRequestContentType = "application/x-www-form-urlencoded";
            //hr.WebRequestReferer = referer;
            HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();

            requestParameters.method      = HttpRequestMethod.Post;
            requestParameters.content     = request;
            requestParameters.contentType = "application/x-www-form-urlencoded";
            requestParameters.referer     = referer;
            Http_v3.LoadUrl(url, requestParameters);
        }
Exemple #5
0
        public T Load(RequestFromWeb_v2 request)
        {
            string url = request.Url;

            if (_urlCache != null)
            {
                string urlPath = _urlCache.GetUrlPath(request.Url, request.RequestParameters);
                if (request.ReloadFromWeb || !zFile.Exists(urlPath))
                {
                    //if (!Http2.LoadToFile(request.Url, urlPath, request.RequestParameters))
                    if (!Http_v3.LoadToFile(request.Url, urlPath, request.RequestParameters))
                    {
                        return(default(T));
                    }
                }
                url = urlPath;
            }
            //if (!Http2.LoadUrl(url, request.RequestParameters))
            if (!Http_v3.LoadUrl(url, request.RequestParameters))
            {
                return(default(T));
            }
            request.LoadResult = true;
            return(_getDataFromWeb(request));
        }
Exemple #6
0
        public bool Load()
        {
            _loadResult = false;
            string url = _url;

            if (_urlCache != null)
            {
                //string urlPath = _urlCache.UrlPath;
                string urlPath = _urlCache.GetUrlPath(url, _requestParameters);
                if (_reload || !zFile.Exists(urlPath))
                {
                    //if (!Http2.LoadToFile(_url, urlPath, _requestParameters))
                    if (!Http_v3.LoadToFile(_url, urlPath, _requestParameters))
                    {
                        return(false);
                    }
                }
                url = urlPath;
            }
            //if (!Http2.LoadUrl(url, _requestParameters))
            if (!Http_v3.LoadUrl(url, _requestParameters))
            {
                return(false);
            }
            _loadResult = true;
            _data       = GetData();
            return(true);
        }
Exemple #7
0
        public static void Test_Handeco_Load_CompanyDetail_01()
        {
            string url = "http://www.handeco.org/fournisseur/consulter/id/1906";

            Http_v3.LoadUrl(url, new HttpRequestParameters_v1 {
                encoding = Encoding.UTF8
            });
            //HtmlXmlReader.CurrentHtmlXmlReader.http.Close();
            //HtmlXmlReader.CurrentHtmlXmlReader.http.Dispose();
        }
Exemple #8
0
        public T Load(RequestFromWeb_v3 request)
        {
            LoadDataFromWeb_v3 loadDataFromWeb = new LoadDataFromWeb_v3 {
                request = request
            };

            DateTime loadFromWebDate;

            string url = request.Url;

            if (_urlCache != null)
            {
                string urlPath = _urlCache.GetUrlPath(url, request.RequestParameters);
                if (request.ReloadFromWeb || !zFile.Exists(urlPath))
                {
                    if (_firstLoadFromWeb && url.StartsWith("http://"))
                    {
                        InitLoadFromWeb();
                        _firstLoadFromWeb = false;
                    }
                    //if (!Http2.LoadToFile(url, urlPath, request.RequestParameters))
                    //    return default(T);
                    if (!Http_v3.LoadToFile(url, urlPath, request.RequestParameters))
                    {
                        return(default(T));
                    }
                }
                url = urlPath;
                // get last write time as loadFromWebDate, dont take creation time because creation time is modified when copying the file
                //loadFromWebDate = new FileInfo(urlPath).LastWriteTime;
                loadFromWebDate = zFile.CreateFileInfo(urlPath).LastWriteTime;
            }
            else
            {
                loadFromWebDate = DateTime.Now;
            }
            if (_firstLoadFromWeb && url.StartsWith("http://"))
            {
                InitLoadFromWeb();
                _firstLoadFromWeb = false;
            }
            //if (!Http2.LoadUrl(url, request.RequestParameters))
            //    return default(T);
            if (!Http_v3.LoadUrl(url, request.RequestParameters))
            {
                return(default(T));
            }
            loadDataFromWeb.loadResult      = true;
            loadDataFromWeb.loadFromWebDate = loadFromWebDate;
            return(GetDataFromWeb(loadDataFromWeb));
        }
        protected MongoImage CreateMongoImage(string url, HttpRequestParameters_v1 requestParameters = null)
        {
            string file = _urlCache.GetUrlSubPath(url, requestParameters);
            string path = zPath.Combine(_urlCache.CacheDirectory, file);

            if (!zFile.Exists(path))
            {
                //Http2.LoadToFile(url, path, requestParameters);
                Http_v3.LoadToFile(url, path, requestParameters);
            }
            Image image = null;

            if (zFile.Exists(path))
            {
                try
                {
                    image = zimg.LoadFromFile(path);
                }
                catch (Exception exception)
                {
                    Trace.WriteLine("error unable to load image url \"{0}\" to \"{1}\" (WebImageMongoCacheManager.CreateMongoImage())", url, path);
                    Trace.Write("error : ");
                    Trace.WriteLine(exception.Message);
                }
            }
            else
            {
                Trace.WriteLine("error unable to load image url \"{0}\" to \"{1}\" (WebImageMongoCacheManager.CreateMongoImage())", url, path);
            }

            MongoImage mongoImage = new MongoImage();

            mongoImage.Url    = url;
            mongoImage.File   = file;
            mongoImage.Width  = image != null ? image.Width : 0;
            mongoImage.Height = image != null ? image.Height : 0;
            mongoImage.Image  = image;

            GetCollection().zUpdate(new QueryDocument {
                { "_id", BsonValue.Create(url) }
            }, new UpdateDocument {
                { "$set", mongoImage.ToBsonDocument() }
            }, UpdateFlags.Upsert);

            return(mongoImage);
        }
Exemple #10
0
        public static void Test_Handeco_LoadHeaderPages_01(int nbPage = 1, bool loadLastPage = false)
        {
            Trace.WriteLine("Test_Handeco_Load_01");
            // www.handeco.org => 87.98.177.160
            // impossible d'aller directement à la page 2
            // les paramètres sont envoyés par un post qui renvoie la page 1
            // puis on fait un get pour les pages suivantes
            string url = "http://www.handeco.org/fournisseurs/rechercher";

            //Http2.LoadUrl(url);

            // tous les départements
            string content = "raisonSociale=&SIRET=&departements%5B%5D=67&departements%5B%5D=68&departements%5B%5D=24&departements%5B%5D=33&departements%5B%5D=40&departements%5B%5D=47&departements%5B%5D=64&departements%5B%5D=03&departements%5B%5D=15&departements%5B%5D=43&departements%5B%5D=63&departements%5B%5D=14&departements%5B%5D=50&departements%5B%5D=61&departements%5B%5D=21&departements%5B%5D=58&departements%5B%5D=71&departements%5B%5D=89&departements%5B%5D=22&departements%5B%5D=29&departements%5B%5D=35&departements%5B%5D=56&departements%5B%5D=18&departements%5B%5D=28&departements%5B%5D=36&departements%5B%5D=37&departements%5B%5D=41&departements%5B%5D=45&departements%5B%5D=08&departements%5B%5D=10&departements%5B%5D=51&departements%5B%5D=52&departements%5B%5D=2A&departements%5B%5D=2B&departements%5B%5D=25&departements%5B%5D=39&departements%5B%5D=70&departements%5B%5D=90&departements%5B%5D=27&departements%5B%5D=76&departements%5B%5D=75&departements%5B%5D=77&departements%5B%5D=78&departements%5B%5D=91&departements%5B%5D=92&departements%5B%5D=93&departements%5B%5D=94&departements%5B%5D=95&departements%5B%5D=11&departements%5B%5D=30&departements%5B%5D=34&departements%5B%5D=48&departements%5B%5D=66&departements%5B%5D=19&departements%5B%5D=23&departements%5B%5D=87&departements%5B%5D=54&departements%5B%5D=55&departements%5B%5D=57&departements%5B%5D=88&departements%5B%5D=09&departements%5B%5D=12&departements%5B%5D=31&departements%5B%5D=32&departements%5B%5D=46&departements%5B%5D=65&departements%5B%5D=81&departements%5B%5D=82&departements%5B%5D=59&departements%5B%5D=62&departements%5B%5D=44&departements%5B%5D=49&departements%5B%5D=53&departements%5B%5D=72&departements%5B%5D=85&departements%5B%5D=02&departements%5B%5D=60&departements%5B%5D=80&departements%5B%5D=16&departements%5B%5D=17&departements%5B%5D=79&departements%5B%5D=86&departements%5B%5D=04&departements%5B%5D=05&departements%5B%5D=06&departements%5B%5D=13&departements%5B%5D=83&departements%5B%5D=84&departements%5B%5D=01&departements%5B%5D=07&departements%5B%5D=26&departements%5B%5D=38&departements%5B%5D=42&departements%5B%5D=69&departements%5B%5D=73&departements%5B%5D=74&departements%5B%5D=971&departements%5B%5D=973&departements%5B%5D=972&departements%5B%5D=974&departements%5B%5D=988&departements%5B%5D=987&departements%5B%5D=975&departements%5B%5D=976&departements%5B%5D=986&experience_cotraitance=0&motsCles=&submitRecherche=Rechercher";
            // Auvergne - 03 Allier
            //string content = "raisonSociale=&SIRET=&departements%5B%5D=03&experience_cotraitance=0&motsCles=&submitRecherche=Rechercher";

            CookieContainer cookies = new CookieContainer();

            HttpRequestParameters_v1 requestParameters = new HttpRequestParameters_v1();

            requestParameters.cookies = cookies;
            requestParameters.method  = HttpRequestMethod.Post;
            requestParameters.content = content;
            //requestParameters.Expect100Continue = true;
            //HtmlXmlReader hxr = HtmlXmlReader.CurrentHtmlXmlReader;
            //hxr.LoadRepeatIfError = 1;
            //hxr.Load(url, requestParameters);
            Http_v3.LoadUrl(url, requestParameters);

            requestParameters         = new HttpRequestParameters_v1();
            requestParameters.cookies = cookies;
            int page = 1;

            for (int i = 1; i < nbPage; i++)
            {
                // http://www.handeco.org/fournisseurs/rechercher/page/2
                url = "http://www.handeco.org/fournisseurs/rechercher/page/" + (++page).ToString();
                Http_v3.LoadUrl(url, requestParameters);
            }

            if (loadLastPage)
            {
                url = "http://www.handeco.org/fournisseurs/rechercher/page/200";
                Http_v3.LoadUrl(url, requestParameters);
            }
        }
Exemple #11
0
        public static void Test_01(string file, bool executeRequest = false)
        {
            Trace.WriteLine("file \"{0}\"", file);
            HttpArchive har = new HttpArchive(file);

            Trace.WriteLine("select first entry");
            var    entry = har.Entries.First();
            string url   = entry.request.GetHttpRequestUrl();

            Trace.WriteLine("url \"{0}\"", url);
            HttpRequestParameters_v1 requestParameters = entry.request.GetHttpRequestParameters();

            Trace.WriteLine("requestParameters :");
            Trace.WriteLine(requestParameters.zToJson());
            if (executeRequest)
            {
                //Http2.LoadUrl(url, requestParameters);
                Http_v3.LoadUrl(url, requestParameters);
            }
        }