Пример #1
0
        // gets string content from a web URL
        private static string RetrieveUrl(string url)
        {
            string pageContents = "";
              // Try to grab the document
              try
              {
            WebGrabber grabber = new WebGrabber(url);
            grabber.Request.Accept = "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
            //grabber.UserAgent = parsedUserAgent;
            grabber.MaxRetries = 10;
            grabber.Timeout = 5000;
            grabber.TimeoutIncrement = 1000;
            //grabber.Encoding = Encoding.UTF8;
            //grabber.Encoding = encoding;
            //grabber.AllowUnsafeHeader = true;
            //grabber.CookieHeader = cookies;
            //grabber.Debug = ScriptSettings.DebugMode;

            // Retrieve the document
            if (grabber.GetResponse())
            {
              pageContents = grabber.GetString();
            }
              }
              catch (Exception e)
              {
            if (e is ThreadAbortException)
              throw e;
            //logger.Warn("Could not connect to " + parsedUrl + ". " + e.Message);
              }
              return pageContents;
        }