/// <summary>
/// Adds a cache item, also replacing if there is already one.
/// </summary>
/// <param name="webResponse">The web response.</param>
/// <param name="fileName">The relative filename.</param>
/// <param name="addToIndex">Whether to add this item to the index.</param>
/// <returns>True for success and false for failure.</returns>
public bool AddCacheItem(HttpWebResponse webResponse, string fileName, bool addToIndex)
{
GlobalCacheItemToAdd newItem = new GlobalCacheItemToAdd(
fileName, webResponse.Headers, (short)webResponse.StatusCode, addToIndex);
// Add file to the disk
if (!AddCacheItemToDisk(webResponse))
{
return false;
}
// Add file to the database (using the method for multiple files that has a try-catch
// and also saves the database.
return AddCacheItemsForExistingFiles(new HashSet<GlobalCacheItemToAdd>() { newItem });
}
/// <summary>
/// Streams a request from the server into the cache.
/// Used for both local and remote proxy requests.
///
/// Replaces existing files.
///
/// Cleans up, but rethrows any exceptions. Also throws own exceptions, if something goes wrong.
/// </summary>
/// <param name="addToIndex">Whether to add this item to the index.</param>
public void DownloadToCache(bool addToIndex)
{
CacheManager cacheManager = _proxy.ProxyCacheManager;
try
{
_proxy.Logger.Debug("downloading: " + _webRequest.RequestUri);
// Stream parameters, if we have non GET/HEAD
HttpUtils.SendBody(_webRequest, _body);
// get the web response for the web request
_webResponse = (HttpWebResponse)_webRequest.GetResponse();
_proxy.Logger.Debug("Received header: " + _webRequest.RequestUri);
if (!_webResponse.ResponseUri.Equals(_webRequest.RequestUri))
{
// redirected at some point
_uriBeforeRedirect = _webRequest.RequestUri.ToString();
// leave a 301 at the old cache file location
string str = "HTTP/1.1 301 Moved Permanently\r\n" +
"Location: " + _webResponse.ResponseUri.ToString() + "\r\n";
// Write the str in a file and save also a 301 entry in the database
// with corresponding header.
cacheManager.CreateFileAndWrite(_cacheFileName, str);
NameValueCollection redirHeaders = new NameValueCollection()
{
{ "Location", _webResponse.ResponseUri.ToString() },
// We need to include content-type, as we always want that header!
{ "Content-Type", "text/plain"}
};
// We won't index redir files.
GlobalCacheItemToAdd newItem = new GlobalCacheItemToAdd(_relCacheFileName, redirHeaders, 301, false);
// Add redir file to the database
cacheManager.AddCacheItemsForExistingFiles(new HashSet<GlobalCacheItemToAdd>() { newItem });
// have to save to the new cache file location
string uri = _webResponse.ResponseUri.ToString();
_relCacheFileName = CacheManager.GetRelativeCacheFileName(uri, _webResponse.Method);
_cacheFileName = _proxy.CachePath + _relCacheFileName;
if (cacheManager.IsCached(_relCacheFileName))
{
_proxy.Logger.Debug("Already exists: " +
_webResponse.Method + " " + uri);
return;
}
}
// Add stream content to the cache.
if (!cacheManager.AddCacheItem(GenericWebResponse, _relCacheFileName, addToIndex))
{
// clean up the (partial) download
cacheManager.RemoveCacheItemFromDisk(_cacheFileName);
_proxy.Logger.Debug("failed, could not add item to the database or cache: " + Uri);
throw new Exception("Could not add item to the database or cache.");
}
}
catch (Exception e)
{
// timed out or some other error
// clean up the (partial) download
cacheManager.RemoveCacheItemFromDisk(_cacheFileName);
_proxy.Logger.Debug("failed: " + Uri, e);
// if this is a webexception, let's throw our own exception include the Remote Proxy's message (on the local side only!):
if (_proxy is RCLocalProxy && e is WebException)
{
WebException exp = e as WebException;
if (exp.Response != null)
{
throw new WebException(exp.Message + ": " + new StreamReader(exp.Response.GetResponseStream()).ReadToEnd(),
exp, exp.Status, exp.Response);
}
}
throw;
}
}
/// <summary>
/// Unpacks the package contents and indexes them. Throws an Exception if anything goes wrong.
/// </summary>
/// <param name="requestHandler">Calling handler for this method.</param>
/// <param name="rcheaders">The rc specific headers.</param>
/// <returns>Total unpacked content size.</returns>
public static long Unpack(LocalRequestHandler requestHandler, RCSpecificResponseHeaders rcheaders)
{
long packageIndexSize = rcheaders.RCPackageIndexSize;
long packageContentSize = rcheaders.RCPackageContentSize;
if (packageIndexSize == 0 || packageContentSize == 0)
{
// This is an internal error that should not happen!
throw new Exception("problem unpacking: package index or content size is 0.");
}
string packageFileName = requestHandler.PackageFileName;
string unpackedPackageFileName = packageFileName.Replace(".gzip", "");
GZipWrapper.GZipDecompress(packageFileName, unpackedPackageFileName, packageIndexSize + packageContentSize);
FileStream packageFs = new FileStream(unpackedPackageFileName, FileMode.Open);
// read the package index
Byte[] packageIndexBuffer = new Byte[packageIndexSize];
int bytesOfIndexRead = 0;
while (bytesOfIndexRead != packageIndexSize)
{
int read = packageFs.Read(packageIndexBuffer,
bytesOfIndexRead, (int)(packageIndexSize - bytesOfIndexRead));
if(read == 0)
{
// This should not happen
throw new Exception("problem unpacking: could not read index.");
}
bytesOfIndexRead += read;
}
// split the big package file into pieces
string[] stringSeparator = new string[] { "\r\n" };
System.Text.UTF8Encoding enc = new System.Text.UTF8Encoding();
string package = enc.GetString(packageIndexBuffer);
string[] packageContentArr = package.Split(stringSeparator, StringSplitOptions.RemoveEmptyEntries);
long unpackedBytes = 0;
HashSet<GlobalCacheItemToAdd> itemsToAdd = new HashSet<GlobalCacheItemToAdd>();
try
{
for (int i = 0; i < packageContentArr.Length; i += 2)
{
// Index format is 2 lines:
// <statusCode> <fileSize> <filename> (filename is last, as it can have spaces)
// <headers> (JSON)
string[] firstLineArray = packageContentArr[i].Split(new string[] { " " }, 3, StringSplitOptions.None);
if (firstLineArray.Length != 3)
{
throw new Exception("unparseable entry: " + packageContentArr[i]);
}
short statusCode;
long fileSize;
try
{
statusCode = Int16.Parse(firstLineArray[0]);
fileSize = Int64.Parse(firstLineArray[1]);
}
catch (Exception)
{
throw new Exception("unparseable entry: " + packageContentArr[i]);
}
string fileName = firstLineArray[2];
string headersJson = packageContentArr[i + 1];
NameValueCollection headers = JsonConvert.DeserializeObject<NameValueCollection>(headersJson,
new NameValueCollectionConverter());
if (requestHandler.Proxy.ProxyCacheManager.CreateOrUpdateFileAndWrite(fileName,
fileSize, packageFs))
{
unpackedBytes += fileSize;
// We index only if (i==0), which means only the first file of the package. This is the main
// page that has been requested. Embedded pages won't be indexed.
GlobalCacheItemToAdd newItem = new GlobalCacheItemToAdd(fileName, headers, statusCode, i == 0);
itemsToAdd.Add(newItem);
}
}
}
finally
{
if (packageFs != null)
{
packageFs.Close();
}
// Add all Database entries
if (!requestHandler.Proxy.ProxyCacheManager.AddCacheItemsForExistingFiles(itemsToAdd))
{
// Adding to the DB failed
throw new Exception("Adding an item to the database failed.");
}
}
return unpackedBytes;
}