Beispiel #1
0
 internal static void CopyAlways(string source, string target)
 {
     if (!File.Exists(source))
     {
         return;
     }
     Directory.CreateDirectory(Path.GetDirectoryName(target));
     IOLibrary.MakeWritable(target);
     File.Copy(source, target, true);
 }
Beispiel #2
0
        private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc)
        {
            string         cachePath = null;
            HttpWebRequest req;
            bool           oldFile = false;

            req        = WebRequest.Create(uri) as HttpWebRequest;
            req.Method = method;

            _fromCache       = false;
            _requestDuration = 0;
            int tc = Environment.TickCount;

            if (UsingCache)
            {
                cachePath = GetCachePath(req.RequestUri);
                if (File.Exists(cachePath))
                {
                    req.IfModifiedSince = File.GetLastAccessTime(cachePath);
                    oldFile             = true;
                }
            }

            if (_cacheOnly)
            {
                if (!File.Exists(cachePath))
                {
                    throw new HtmlWebException("File was not found at cache path: '" + cachePath + "'");
                }

                if (path != null)
                {
                    IOLibrary.CopyAlways(cachePath, path);
                    // touch the file
                    File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
                }
                _fromCache = true;
                return(HttpStatusCode.NotModified);
            }

            if (_useCookies)
            {
                req.CookieContainer = new CookieContainer();
            }

            if (PreRequest != null)
            {
                // allow our user to change the request at will
                if (!PreRequest(req))
                {
                    return(HttpStatusCode.ResetContent);
                }

                // dump cookie
//				if (_useCookies)
//				{
//					foreach(Cookie cookie in req.CookieContainer.GetCookies(req.RequestUri))
//					{
//						HtmlLibrary.Trace("Cookie " + cookie.Name + "=" + cookie.Value + " path=" + cookie.Path + " domain=" + cookie.Domain);
//					}
//				}
            }

            HttpWebResponse resp;

            try
            {
                resp = req.GetResponse() as HttpWebResponse;
            }
            catch (WebException we)
            {
                _requestDuration = Environment.TickCount - tc;
                resp             = (HttpWebResponse)we.Response;
                if (resp == null)
                {
                    if (oldFile)
                    {
                        if (path != null)
                        {
                            IOLibrary.CopyAlways(cachePath, path);
                            // touch the file
                            File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
                        }
                        return(HttpStatusCode.NotModified);
                    }
                    throw;
                }
            }
            catch (Exception)
            {
                _requestDuration = Environment.TickCount - tc;
                throw;
            }

            // allow our user to get some info from the response
            if (PostResponse != null)
            {
                PostResponse(req, resp);
            }

            _requestDuration = Environment.TickCount - tc;
            _responseUri     = resp.ResponseUri;

            bool html = IsHtmlContent(resp.ContentType);

            System.Text.Encoding respenc;

            /// 芦侃修改 2008-7-11 15:12:35
            try
            {
                respenc = System.Text.Encoding.GetEncoding(resp.ContentEncoding);
            }
            catch (Exception)
            {
                try
                {
                    switch (resp.CharacterSet.ToLower())
                    {
                    case "iso-8859-1":
                        respenc = Encoding.Default;
                        break;

                    default:
                        respenc = System.Text.Encoding.GetEncoding(resp.CharacterSet);
                        break;
                    }
                }
                catch (Exception)
                {
                    respenc = Encoding.Default;
                }
            }
            Stream s;

            if (resp.ContentEncoding.ToLower().Equals("gzip"))
            {
                /// 如果页面采用了GZip进行压缩,调用.net的GZip类进行解压
                s = new GZipStream(resp.GetResponseStream(), CompressionMode.Decompress);
            }
            else
            {
                s = resp.GetResponseStream();
            }
            /**********芦侃修改结束**********/

            if (resp.StatusCode == HttpStatusCode.NotModified)
            {
                if (UsingCache)
                {
                    _fromCache = true;
                    if (path != null)
                    {
                        IOLibrary.CopyAlways(cachePath, path);
                        // touch the file
                        File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
                    }
                    return(resp.StatusCode);
                }
                else
                {
                    // this should *never* happen...
                    throw new HtmlWebException("Server has send a NotModifed code, without cache enabled.");
                }
            }
            if (s != null)
            {
                if (UsingCache)
                {
                    // NOTE: LastModified does not contain milliseconds, so we remove them to the file
                    SaveStream(s, cachePath, RemoveMilliseconds(resp.LastModified), _streamBufferSize);

                    // save headers
                    SaveCacheHeaders(req.RequestUri, resp);

                    if (path != null)
                    {
                        // copy and touch the file
                        IOLibrary.CopyAlways(cachePath, path);
                        File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
                    }
                }
                else
                {
                    // try to work in-memory
                    if ((doc != null) && (html))
                    {
                        if (respenc != null)
                        {
                            doc.Load(s, respenc);
                        }
                        else
                        {
                            doc.Load(s, true);
                        }
                    }
                }
                resp.Close();
            }
            return(resp.StatusCode);
        }
Beispiel #3
0
        private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc, IWebProxy proxy,
                                   ICredentials creds)
        {
            string         cachePath = null;
            HttpWebRequest req;
            bool           oldFile = false;

            req        = WebRequest.Create(uri) as HttpWebRequest;
            req.Method = method;

            if (proxy != null)
            {
                if (creds != null)
                {
                    proxy.Credentials = creds;
                    req.Credentials   = creds;
                }
                else
                {
                    proxy.Credentials = CredentialCache.DefaultCredentials;
                    req.Credentials   = CredentialCache.DefaultCredentials;
                }
                req.Proxy = proxy;
            }

            _fromCache       = false;
            _requestDuration = 0;
            int tc = Environment.TickCount;

            if (UsingCache)
            {
                cachePath = GetCachePath(req.RequestUri);
                if (File.Exists(cachePath))
                {
                    req.IfModifiedSince = File.GetLastAccessTime(cachePath);
                    oldFile             = true;
                }
            }

            if (_cacheOnly)
            {
                if (!File.Exists(cachePath))
                {
                    throw new HtmlWebException("File was not found at cache path: '" + cachePath + "'");
                }

                if (path != null)
                {
                    IOLibrary.CopyAlways(cachePath, path);
                    // touch the file
                    File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
                }
                _fromCache = true;
                return(HttpStatusCode.NotModified);
            }

            if (_useCookies)
            {
                req.CookieContainer = new CookieContainer();
            }

            if (PreRequest != null)
            {
                // allow our user to change the request at will
                if (!PreRequest(req))
                {
                    return(HttpStatusCode.ResetContent);
                }

                // dump cookie
                //				if (_useCookies)
                //				{
                //					foreach(Cookie cookie in req.CookieContainer.GetCookies(req.RequestUri))
                //					{
                //						HtmlLibrary.Trace("Cookie " + cookie.Name + "=" + cookie.Value + " path=" + cookie.Path + " domain=" + cookie.Domain);
                //					}
                //				}
            }

            HttpWebResponse resp;

            try
            {
                resp = req.GetResponse() as HttpWebResponse;
            }
            catch (WebException we)
            {
                _requestDuration = Environment.TickCount - tc;
                resp             = (HttpWebResponse)we.Response;
                if (resp == null)
                {
                    if (oldFile)
                    {
                        if (path != null)
                        {
                            IOLibrary.CopyAlways(cachePath, path);
                            // touch the file
                            File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
                        }
                        return(HttpStatusCode.NotModified);
                    }
                    throw;
                }
            }
            catch (Exception)
            {
                _requestDuration = Environment.TickCount - tc;
                throw;
            }

            // allow our user to get some info from the response
            if (PostResponse != null)
            {
                PostResponse(req, resp);
            }

            _requestDuration = Environment.TickCount - tc;
            _responseUri     = resp.ResponseUri;

            bool     html = IsHtmlContent(resp.ContentType);
            Encoding respenc;

            if ((resp.ContentEncoding != null) && (resp.ContentEncoding.Length > 0))
            {
                respenc = Encoding.GetEncoding(resp.ContentEncoding);
            }
            else
            {
                respenc = null;
            }

            if (resp.StatusCode == HttpStatusCode.NotModified)
            {
                if (UsingCache)
                {
                    _fromCache = true;
                    if (path != null)
                    {
                        IOLibrary.CopyAlways(cachePath, path);
                        // touch the file
                        File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
                    }
                    return(resp.StatusCode);
                }
                else
                {
                    // this should *never* happen...
                    throw new HtmlWebException("Server has send a NotModifed code, without cache enabled.");
                }
            }
            Stream s = resp.GetResponseStream();

            if (s != null)
            {
                if (UsingCache)
                {
                    // NOTE: LastModified does not contain milliseconds, so we remove them to the file
                    SaveStream(s, cachePath, RemoveMilliseconds(resp.LastModified), _streamBufferSize);

                    // save headers
                    SaveCacheHeaders(req.RequestUri, resp);

                    if (path != null)
                    {
                        // copy and touch the file
                        IOLibrary.CopyAlways(cachePath, path);
                        File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
                    }
                }
                else
                {
                    // try to work in-memory
                    if ((doc != null) && (html))
                    {
                        if (respenc != null)
                        {
                            doc.Load(s, respenc);
                        }
                        else
                        {
                            doc.Load(s, true);
                        }
                    }
                }
                resp.Close();
            }
            return(resp.StatusCode);
        }