Exemplo n.º 1
0
 private XmlNodeList GetXml(string url)
 {
     // given a url, retrieves the xml result set and returns the nodelist of Item objects
       WebGrabber grabber = new WebGrabber(url)
       {
     MaxRetries = 10,
     Timeout = 5000,
     TimeoutIncrement = 1000,
     Encoding = Encoding.UTF8
       };
       return grabber.GetResponse() ? grabber.GetXML() : null;
 }
Exemplo n.º 2
0
        // gets string content from a web URL
        private static string RetrieveUrl(string url)
        {
            string pageContents = "";
              // Try to grab the document
              try
              {
            WebGrabber grabber = new WebGrabber(url);
            grabber.Request.Accept = "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
            //grabber.UserAgent = parsedUserAgent;
            grabber.MaxRetries = 10;
            grabber.Timeout = 5000;
            grabber.TimeoutIncrement = 1000;
            //grabber.Encoding = Encoding.UTF8;
            //grabber.Encoding = encoding;
            //grabber.AllowUnsafeHeader = true;
            //grabber.CookieHeader = cookies;
            //grabber.Debug = ScriptSettings.DebugMode;

            // Retrieve the document
            if (grabber.GetResponse())
            {
              pageContents = grabber.GetString();
            }
              }
              catch (Exception e)
              {
            if (e is ThreadAbortException)
              throw e;
            //logger.Warn("Could not connect to " + parsedUrl + ". " + e.Message);
              }
              return pageContents;
        }