/// <summary>
/// Adds a new cache item to the database. Throws an exception if anything goes wrong.
///
/// databaseContext.SaveChanges() still needs to be called.
/// </summary>
/// <param name="filename">The filename.</param>
/// <param name="headers">The headers.</param>
/// <param name="statusCode">The status code.</param>
/// <param name="fileSize">The file size.</param>
/// <param name="addToIndex">Whether a text/html file should be added to the Lucene index.</param>
/// <param name="databaseContext">The database context.</param>
private void AddCacheItemToDatabase(string filename,
NameValueCollection headers, short statusCode,long fileSize, bool addToIndex,
RCDatabaseEntities databaseContext)
{
// We disable cookies for non-streamed requests
headers.Remove("Set-Cookie");
string headersJson = JsonConvert.SerializeObject(headers,
Formatting.None, new NameValueCollectionConverter());
_proxy.Logger.Debug("Adding to database: " + filename);
// Check if the RC data still exists (this means the file has been cached previsouly and was evicted)
GlobalCacheRCData rcData = GetGlobalCacheRCData(filename, databaseContext);
if (rcData == null)
{
// create a new rc data item
rcData = new GlobalCacheRCData();
// Save the rc values
rcData.filename = filename;
// Although this is not really a request, we set the lastRequestTime to now
rcData.lastRequestTime = DateTime.Now;
// One requests so far
rcData.numberOfRequests = 1;
// Download time is the lastModified time of the file, if it already exists. Otherwise now
rcData.downloadTime = File.Exists(_cachePath + filename) ?
File.GetLastWriteTime(_cachePath + filename) : DateTime.Now;
// add item
databaseContext.GlobalCacheRCData.Add(rcData);
}
else
{
// Update RC data
// Although this is not really a request, we set the lastRequestTime to now
rcData.lastRequestTime = DateTime.Now;
// One request more
rcData.numberOfRequests++;
// Download time is the lastModified time of the file, if it already exists. Otherwise now
rcData.downloadTime = File.Exists(_cachePath + filename) ?
File.GetLastWriteTime(_cachePath + filename) : DateTime.Now;
}
// Create item and save the values.
GlobalCacheItem cacheItem = new GlobalCacheItem();
cacheItem.responseHeaders = headersJson;
cacheItem.statusCode = statusCode;
cacheItem.filename = filename;
cacheItem.filesize = fileSize;
// add item
databaseContext.GlobalCacheItem.Add(cacheItem);
// If we're on the local proxy, we want to add text documents to the Lucene index.
if (addToIndex && _proxy is RCLocalProxy && GetHTTPMethodFromRelCacheFileName(filename).Equals("GET") &&
(headers["Content-Type"].Contains("text/html") || headers["Content-Type"].Contains("text/plain")))
{
RCLocalProxy proxy = ((RCLocalProxy)_proxy);
// The index might not have been initialized...
if (proxy.IndexWrapper == null)
{
// FIXME We should not use the Program var here.
// But when we're creating the DB, this gets called in the RCProxy constructor
// before the RCLocalProxy constructor. We should find a way to have the index created
// before the cache for the LP
proxy.IndexWrapper = new IndexWrapper(Program.INDEX_PATH);
// initialize the index
proxy.IndexWrapper.EnsureIndexExists();
}
// We have made sure the content-type header is always present in the DB!
// XXX reading the file we just wrote. Not perfect.
string document = Utils.ReadFileAsString(_cachePath + filename);
string title = HtmlUtils.GetPageTitleFromHTML(document);
// Use whole document, so we can also find results with tags, etc.
try
{
proxy.Logger.Debug("Adding to index: " + filename);
proxy.IndexWrapper.IndexDocument(FilePathToUri(filename), title, document);
}
catch (Exception e)
{
_proxy.Logger.Warn("Could not add document to index.", e);
}
}
}
/// <summary>
/// Updates a cache item in the database. Throws an exception if anything goes wrong.
///
/// databaseContext.SaveChanges() still needs to be called.
/// </summary>
/// <param name="existingCacheItem">The existing global cache item.</param>
/// <param name="headers">the new headers</param>
/// <param name="statusCode">The new status code.</param>
/// <param name="fileSize">The file size.</param>
/// <param name="databaseContext">The database context.</param>
private void UpdateCacheItemInDatabase(GlobalCacheItem existingCacheItem, NameValueCollection headers,
short statusCode, long fileSize, RCDatabaseEntities databaseContext)
{
string headersJson = JsonConvert.SerializeObject(headers,
Formatting.None, new NameValueCollectionConverter());
_proxy.Logger.Debug("Updating in database: " + existingCacheItem.filename);
// Update non-RC data
existingCacheItem.responseHeaders = headersJson;
existingCacheItem.statusCode = statusCode;
existingCacheItem.filesize = fileSize;
// Update RC data
GlobalCacheRCData rcData = existingCacheItem.GlobalCacheRCData;
// Although this is not really a request, we set the lastRequestTime to now
rcData.lastRequestTime = DateTime.Now;
// One request more
rcData.numberOfRequests++;
// Download time is the lastModified time of the file, if it already exists. Otherwise now
rcData.downloadTime = File.Exists(_cachePath + existingCacheItem.filename) ?
File.GetLastWriteTime(_cachePath + existingCacheItem.filename) : DateTime.Now;
}
/// <summary>
/// Adds a cache item to the database. The file is assumed to exist already. If that item exists
/// already in the DB, it will be replaced with the new headers and statusCode, and the RC data will
/// be updated.
///
/// If the item did not exist and it does not fit in the cache, other items will be evicted.
/// </summary>
/// <param name="relFileName">The filename.</param>
/// <param name="headers">The headers.</param>
/// <param name="statusCode">The status code.</param>
/// <param name="addToIndex">Whether a text/html file should be added to the Lucene index.</param>
/// <param name="databaseContext">The database context.</param>
/// <returns>True for success and false for failure.</returns>
private bool AddCacheItemForExistingFile(string relFileName,
NameValueCollection headers, short statusCode, bool addToIndex, RCDatabaseEntities databaseContext)
{
// If the headers do not contain "Content-Type", which should practically not happen,
// (but servers are actually not required to send it) we set it to the default:
// "application/octet-stream"
if (headers["Content-Type"] == null)
{
headers["Content-Type"] = "application/octet-stream";
}
long itemSize;
// Get the cache and the file size
try
{
itemSize = CacheItemFileSize(relFileName);
}
catch (Exception e)
{
_proxy.Logger.Warn("Could not compute file size.", e);
return false;
}
GlobalCacheItem existingCacheItem = GetGlobalCacheItem(relFileName, databaseContext);
// Look if we have to evict cache items first.
long cacheOversize = _cacheSize + itemSize - _maxCacheSize;
if (existingCacheItem != null)
{
cacheOversize -= -existingCacheItem.filesize;
}
if (cacheOversize > 0)
{
try
{
// We have to evict. We do until 5 % is free again.
EvictCacheItems((long)(cacheOversize + CACHE_EVICTION_PERCENT * _maxCacheSize), databaseContext);
// Always save after evicting, otherwise the CacheSize will not return the correct value
// the next time and we will try evicting the same items again, which produces errors.
databaseContext.SaveChanges();
}
catch (Exception e)
{
_proxy.Logger.Warn("Could not evict cache items.", e);
return false;
}
}
if (existingCacheItem == null)
{
// Add database entry.
try
{
AddCacheItemToDatabase(relFileName, headers, statusCode,
itemSize, addToIndex, databaseContext);
}
catch (Exception e)
{
_proxy.Logger.Warn("Could not add cache item to the database.", e);
return false;
}
}
else
{
// Update database entry.
try
{
UpdateCacheItemInDatabase(existingCacheItem, headers, statusCode,
itemSize, databaseContext);
}
catch (Exception e)
{
_proxy.Logger.Warn("Could not add cache item to the database.", e);
return false;
}
}
return true;
}
/// <summary>
/// Checks whether an item is cached. Only the DB is being used,
/// the disk contens are not ebing looked at.
/// </summary>
/// <param name="filename">The filename.</param>
/// <param name="databaseContext">The database context.</param>
/// <returns>If the item is cached.</returns>
private bool IsCached(string filename, RCDatabaseEntities databaseContext)
{
if (filename.Length > 260)
{
return false;
}
return (from gci in databaseContext.GlobalCacheItem
where gci.filename.Equals(filename)
select 1).Count() != 0;
}
/// <summary>
/// Removes a cache item and deletes the file. This method is used internally, when evicting.
/// </summary>
/// <param name="cacheItem">The cache item.</param>
/// <param name="databaseContext">The database context.</param>
private void RemoveCacheItem(GlobalCacheItem cacheItem, RCDatabaseEntities databaseContext)
{
_proxy.Logger.Debug(String.Format("Removing from the cache: {0} Last request: {1}",
cacheItem.filename, cacheItem.GlobalCacheRCData.lastRequestTime));
// Remove file
Utils.DeleteFile(_cachePath + cacheItem.filename);
// Remove cache item entry
databaseContext.GlobalCacheItem.Remove(cacheItem);
// If we're on the local proxy, we want to remove text documents from the Lucene index.
if (_proxy is RCLocalProxy && GetHTTPMethodFromRelCacheFileName(cacheItem.filename).Equals("GET") &&
(cacheItem.responseHeaders.Contains("\"Content-Type\":[\"text/html") ||
cacheItem.responseHeaders.Contains("\"Content-Type\":[\"text/plain")))
{
try
{
// remove the file from Lucene, if it is a GET text or HTML file.
// We have made sure the content-type header is always present in the DB!
((RCLocalProxy)_proxy).IndexWrapper.DeleteDocument(FilePathToUri(cacheItem.filename));
}
catch (Exception e)
{
_proxy.Logger.Warn("Could not remove document from the index.", e);
}
}
}
/// <summary>
/// Gets a new database context. This context must not be shared among threads!
///
/// The context will never detect changes (which improves performance), as we have
/// our own synchronization.
/// </summary>
/// <param name="autoDetectChangesEnabled">Whether this database context should detect changes to
/// attached items and write them back when saveChanges is called. As we have our own synchronization,
/// this should only be set to true, if we update an item. Otherwise performance will be bad.</param>
/// <returns>A new database context.</returns>
private RCDatabaseEntities GetNewDatabaseContext(bool autoDetectChangesEnabled)
{
// Create context and modify connection string to point to our DB file.
RCDatabaseEntities result = new RCDatabaseEntities();
result.Database.Connection.ConnectionString =
String.Format("data source=\"{0}\";Max Database Size={1};Max Buffer Size={2}",
_proxy.ProxyPath + DATABASE_FILE_NAME, DATABASE_MAX_SIZE_MB, DATABASE_BUFFER_MAX_SIZE_KB);
result.Configuration.AutoDetectChangesEnabled = autoDetectChangesEnabled;
result.Configuration.ValidateOnSaveEnabled = false;
return result;
}
/// <summary>
/// Gets the global cache RC data item for the specified HTTP method and URI, if it exists,
/// and null otherwise.
/// </summary>
/// <param name="filename">The filename.</param>
/// <param name="databaseContext">The database context.</param>
/// <returns>The global cache RC data item or null.</returns>
private GlobalCacheRCData GetGlobalCacheRCData(string filename, RCDatabaseEntities databaseContext)
{
if (filename.Length > 260)
{
return null;
}
return (from gcrc in databaseContext.GlobalCacheRCData
where gcrc.filename.Equals(filename)
select gcrc).FirstOrDefault();
}
/// <summary>
/// Gets a global cache item and updates the rc metadata.
///
/// databaseContext.SaveChanges() still needs to be called.
/// </summary>
/// <param name="filename">The filename.</param>
/// <param name="databaseContext">The database context.</param>
/// <returns>The item or null.</returns>
private GlobalCacheItem GetGlobalCacheItemAsRequest(string filename, RCDatabaseEntities databaseContext)
{
GlobalCacheItem result = GetGlobalCacheItem(filename, databaseContext);
if (result != null)
{
_proxy.Logger.Debug(String.Format("Updating request time and number of requests of {0}",
filename));
// Modify the RC data
result.GlobalCacheRCData.lastRequestTime = DateTime.Now;
result.GlobalCacheRCData.numberOfRequests++;
}
return result;
}
/// <summary>
/// Evict items from the cache (also deleting the files from disk), until
/// a total of bytesToEvict is deleted.
///
/// LRU is the eviction strategy.
/// </summary>
/// <param name="bytesToEvict">The number of bytes to evict.</param>
/// <param name="databaseContext">The database context.</param>
private void EvictCacheItems(long bytesToEvict, RCDatabaseEntities databaseContext)
{
_proxy.Logger.Debug(String.Format("Evicting {0} bytes from the cache.", bytesToEvict));
long evicted = 0;
IOrderedQueryable<GlobalCacheItem> orderedCacheItems = (from gci in databaseContext.GlobalCacheItem select gci).
OrderBy(gci => gci.GlobalCacheRCData.lastRequestTime);
foreach (GlobalCacheItem gci in orderedCacheItems)
{
evicted += gci.filesize;
RemoveCacheItem(gci, databaseContext);
if (evicted >= bytesToEvict)
{
break;
}
}
}
/// <summary>
/// Gets the number of cached items. There will be more GlobalCacheRCData items than this.
/// </summary>
/// <param name="databaseContext">The database context.</param>
/// <returns>The number of cached items.</returns>
private int CachedItems(RCDatabaseEntities databaseContext)
{
return (from gci in databaseContext.GlobalCacheItem select 1).Count();
}
