Exemple #1
0
        /// <summary>
        /// Adds a cache item, also replacing if there is already one.
        /// </summary>
        /// <param name="webResponse">The web response.</param>
        /// <param name="fileName">The relative filename.</param>
        /// <param name="addToIndex">Whether to add this item to the index.</param>
        /// <returns>True for success and false for failure.</returns>
        public bool AddCacheItem(HttpWebResponse webResponse, string fileName, bool addToIndex)
        {
            GlobalCacheItemToAdd newItem = new GlobalCacheItemToAdd(
                fileName, webResponse.Headers, (short)webResponse.StatusCode, addToIndex);

            // Add file to the disk
            if (!AddCacheItemToDisk(webResponse))
            {
                return false;
            }
            // Add file to the database (using the method for multiple files that has a try-catch
            // and also saves the database.
            return AddCacheItemsForExistingFiles(new HashSet<GlobalCacheItemToAdd>() { newItem });
        }
Exemple #2
0
        /// <summary>
        /// Streams a request from the server into the cache.
        /// Used for both local and remote proxy requests.
        /// 
        /// Replaces existing files.
        /// 
        /// Cleans up, but rethrows any exceptions. Also throws own exceptions, if something goes wrong.
        /// </summary>
        /// <param name="addToIndex">Whether to add this item to the index.</param>
        public void DownloadToCache(bool addToIndex)
        {
            CacheManager cacheManager = _proxy.ProxyCacheManager;

            try
            {
                _proxy.Logger.Debug("downloading: " + _webRequest.RequestUri);

                // Stream parameters, if we have non GET/HEAD
                HttpUtils.SendBody(_webRequest, _body);

                // get the web response for the web request
                _webResponse = (HttpWebResponse)_webRequest.GetResponse();
                _proxy.Logger.Debug("Received header: " + _webRequest.RequestUri);

                if (!_webResponse.ResponseUri.Equals(_webRequest.RequestUri))
                {
                    // redirected at some point
                    _uriBeforeRedirect = _webRequest.RequestUri.ToString();

                    // leave a 301 at the old cache file location
                    string str = "HTTP/1.1 301 Moved Permanently\r\n" +
                          "Location: " + _webResponse.ResponseUri.ToString() + "\r\n";

                    // Write the str in a file and save also a 301 entry in the database
                    // with corresponding header.
                    cacheManager.CreateFileAndWrite(_cacheFileName, str);
                    NameValueCollection redirHeaders = new NameValueCollection()
                    {
                        { "Location", _webResponse.ResponseUri.ToString() },
                        // We need to include content-type, as we always want that header!
                        { "Content-Type", "text/plain"}
                    };
                    // We won't index redir files.
                    GlobalCacheItemToAdd newItem = new GlobalCacheItemToAdd(_relCacheFileName, redirHeaders, 301, false);

                    // Add redir file to the database
                    cacheManager.AddCacheItemsForExistingFiles(new HashSet<GlobalCacheItemToAdd>() { newItem });

                    // have to save to the new cache file location
                    string uri = _webResponse.ResponseUri.ToString();
                    _relCacheFileName = CacheManager.GetRelativeCacheFileName(uri, _webResponse.Method);
                    _cacheFileName = _proxy.CachePath + _relCacheFileName;

                    if (cacheManager.IsCached(_relCacheFileName))
                    {
                        _proxy.Logger.Debug("Already exists: " +
                            _webResponse.Method + " " + uri);
                        return;
                    }
                }

                // Add stream content to the cache.
                if (!cacheManager.AddCacheItem(GenericWebResponse, _relCacheFileName, addToIndex))
                {
                    // clean up the (partial) download
                    cacheManager.RemoveCacheItemFromDisk(_cacheFileName);
                    _proxy.Logger.Debug("failed, could not add item to the database or cache: " + Uri);
                    throw new Exception("Could not add item to the database or cache.");
                }
            }
            catch (Exception e)
            {
                // timed out or some other error
                // clean up the (partial) download
                cacheManager.RemoveCacheItemFromDisk(_cacheFileName);
                _proxy.Logger.Debug("failed: " + Uri, e);

                // if this is a webexception, let's throw our own exception include the Remote Proxy's message (on the local side only!):
                if (_proxy is RCLocalProxy && e is WebException)
                {
                    WebException exp = e as WebException;
                    if (exp.Response != null)
                    {
                        throw new WebException(exp.Message + ": " + new StreamReader(exp.Response.GetResponseStream()).ReadToEnd(),
                            exp, exp.Status, exp.Response);
                    }
                }

                throw;
            }
        }
Exemple #3
0
        /// <summary>
        /// Unpacks the package contents and indexes them. Throws an Exception if anything goes wrong.
        /// </summary>
        /// <param name="requestHandler">Calling handler for this method.</param>
        /// <param name="rcheaders">The rc specific headers.</param>
        /// <returns>Total unpacked content size.</returns>
        public static long Unpack(LocalRequestHandler requestHandler, RCSpecificResponseHeaders rcheaders)
        {
            long packageIndexSize = rcheaders.RCPackageIndexSize;
            long packageContentSize = rcheaders.RCPackageContentSize;
            if (packageIndexSize == 0 || packageContentSize == 0)
            {
                // This is an internal error that should not happen!
                throw new Exception("problem unpacking: package index or content size is 0.");
            }

            string packageFileName = requestHandler.PackageFileName;
            string unpackedPackageFileName = packageFileName.Replace(".gzip", "");

            GZipWrapper.GZipDecompress(packageFileName, unpackedPackageFileName, packageIndexSize + packageContentSize);
            FileStream packageFs = new FileStream(unpackedPackageFileName, FileMode.Open);

            // read the package index
            Byte[] packageIndexBuffer = new Byte[packageIndexSize];
            int bytesOfIndexRead = 0;
            while (bytesOfIndexRead != packageIndexSize)
            {
                int read = packageFs.Read(packageIndexBuffer,
                    bytesOfIndexRead, (int)(packageIndexSize - bytesOfIndexRead));
                if(read == 0)
                {
                    // This should not happen
                    throw new Exception("problem unpacking: could not read index.");
                }
                bytesOfIndexRead += read;
            }

            // split the big package file into pieces
            string[] stringSeparator = new string[] { "\r\n" };
            System.Text.UTF8Encoding enc = new System.Text.UTF8Encoding();
            string package = enc.GetString(packageIndexBuffer);
            string[] packageContentArr = package.Split(stringSeparator, StringSplitOptions.RemoveEmptyEntries);

            long unpackedBytes = 0;
            HashSet<GlobalCacheItemToAdd> itemsToAdd = new HashSet<GlobalCacheItemToAdd>();

            try
            {
                for (int i = 0; i < packageContentArr.Length; i += 2)
                {
                    // Index format is 2 lines:
                    // <statusCode> <fileSize> <filename> (filename is last, as it can have spaces)
                    // <headers> (JSON)
                    string[] firstLineArray = packageContentArr[i].Split(new string[] { " " }, 3, StringSplitOptions.None);

                    if (firstLineArray.Length != 3)
                    {
                        throw new Exception("unparseable entry: " + packageContentArr[i]);
                    }
                    short statusCode;
                    long fileSize;
                    try
                    {
                        statusCode = Int16.Parse(firstLineArray[0]);
                        fileSize = Int64.Parse(firstLineArray[1]);
                    }
                    catch (Exception)
                    {
                        throw new Exception("unparseable entry: " + packageContentArr[i]);
                    }
                    string fileName = firstLineArray[2];

                    string headersJson = packageContentArr[i + 1];
                    NameValueCollection headers = JsonConvert.DeserializeObject<NameValueCollection>(headersJson,
                        new NameValueCollectionConverter());

                    if (requestHandler.Proxy.ProxyCacheManager.CreateOrUpdateFileAndWrite(fileName,
                        fileSize, packageFs))
                    {
                        unpackedBytes += fileSize;
                        // We index only if (i==0), which means only the first file of the package. This is the main
                        // page that has been requested. Embedded pages won't be indexed.
                        GlobalCacheItemToAdd newItem = new GlobalCacheItemToAdd(fileName, headers, statusCode, i == 0);
                        itemsToAdd.Add(newItem);
                    }
                }
            }
            finally
            {
                if (packageFs != null)
                {
                    packageFs.Close();
                }
                // Add all Database entries
                if (!requestHandler.Proxy.ProxyCacheManager.AddCacheItemsForExistingFiles(itemsToAdd))
                {
                    // Adding to the DB failed
                    throw new Exception("Adding an item to the database failed.");
                }
            }

            return unpackedBytes;
        }