/// <summary> /// Adds a new cache item to the database. Throws an exception if anything goes wrong. /// /// databaseContext.SaveChanges() still needs to be called. /// </summary> /// <param name="filename">The filename.</param> /// <param name="headers">The headers.</param> /// <param name="statusCode">The status code.</param> /// <param name="fileSize">The file size.</param> /// <param name="addToIndex">Whether a text/html file should be added to the Lucene index.</param> /// <param name="databaseContext">The database context.</param> private void AddCacheItemToDatabase(string filename, NameValueCollection headers, short statusCode,long fileSize, bool addToIndex, RCDatabaseEntities databaseContext) { // We disable cookies for non-streamed requests headers.Remove("Set-Cookie"); string headersJson = JsonConvert.SerializeObject(headers, Formatting.None, new NameValueCollectionConverter()); _proxy.Logger.Debug("Adding to database: " + filename); // Check if the RC data still exists (this means the file has been cached previsouly and was evicted) GlobalCacheRCData rcData = GetGlobalCacheRCData(filename, databaseContext); if (rcData == null) { // create a new rc data item rcData = new GlobalCacheRCData(); // Save the rc values rcData.filename = filename; // Although this is not really a request, we set the lastRequestTime to now rcData.lastRequestTime = DateTime.Now; // One requests so far rcData.numberOfRequests = 1; // Download time is the lastModified time of the file, if it already exists. Otherwise now rcData.downloadTime = File.Exists(_cachePath + filename) ? File.GetLastWriteTime(_cachePath + filename) : DateTime.Now; // add item databaseContext.GlobalCacheRCData.Add(rcData); } else { // Update RC data // Although this is not really a request, we set the lastRequestTime to now rcData.lastRequestTime = DateTime.Now; // One request more rcData.numberOfRequests++; // Download time is the lastModified time of the file, if it already exists. Otherwise now rcData.downloadTime = File.Exists(_cachePath + filename) ? File.GetLastWriteTime(_cachePath + filename) : DateTime.Now; } // Create item and save the values. GlobalCacheItem cacheItem = new GlobalCacheItem(); cacheItem.responseHeaders = headersJson; cacheItem.statusCode = statusCode; cacheItem.filename = filename; cacheItem.filesize = fileSize; // add item databaseContext.GlobalCacheItem.Add(cacheItem); // If we're on the local proxy, we want to add text documents to the Lucene index. if (addToIndex && _proxy is RCLocalProxy && GetHTTPMethodFromRelCacheFileName(filename).Equals("GET") && (headers["Content-Type"].Contains("text/html") || headers["Content-Type"].Contains("text/plain"))) { RCLocalProxy proxy = ((RCLocalProxy)_proxy); // The index might not have been initialized... if (proxy.IndexWrapper == null) { // FIXME We should not use the Program var here. // But when we're creating the DB, this gets called in the RCProxy constructor // before the RCLocalProxy constructor. We should find a way to have the index created // before the cache for the LP proxy.IndexWrapper = new IndexWrapper(Program.INDEX_PATH); // initialize the index proxy.IndexWrapper.EnsureIndexExists(); } // We have made sure the content-type header is always present in the DB! // XXX reading the file we just wrote. Not perfect. string document = Utils.ReadFileAsString(_cachePath + filename); string title = HtmlUtils.GetPageTitleFromHTML(document); // Use whole document, so we can also find results with tags, etc. try { proxy.Logger.Debug("Adding to index: " + filename); proxy.IndexWrapper.IndexDocument(FilePathToUri(filename), title, document); } catch (Exception e) { _proxy.Logger.Warn("Could not add document to index.", e); } } }
/// <summary> /// Updates a cache item in the database. Throws an exception if anything goes wrong. /// /// databaseContext.SaveChanges() still needs to be called. /// </summary> /// <param name="existingCacheItem">The existing global cache item.</param> /// <param name="headers">the new headers</param> /// <param name="statusCode">The new status code.</param> /// <param name="fileSize">The file size.</param> /// <param name="databaseContext">The database context.</param> private void UpdateCacheItemInDatabase(GlobalCacheItem existingCacheItem, NameValueCollection headers, short statusCode, long fileSize, RCDatabaseEntities databaseContext) { string headersJson = JsonConvert.SerializeObject(headers, Formatting.None, new NameValueCollectionConverter()); _proxy.Logger.Debug("Updating in database: " + existingCacheItem.filename); // Update non-RC data existingCacheItem.responseHeaders = headersJson; existingCacheItem.statusCode = statusCode; existingCacheItem.filesize = fileSize; // Update RC data GlobalCacheRCData rcData = existingCacheItem.GlobalCacheRCData; // Although this is not really a request, we set the lastRequestTime to now rcData.lastRequestTime = DateTime.Now; // One request more rcData.numberOfRequests++; // Download time is the lastModified time of the file, if it already exists. Otherwise now rcData.downloadTime = File.Exists(_cachePath + existingCacheItem.filename) ? File.GetLastWriteTime(_cachePath + existingCacheItem.filename) : DateTime.Now; }
/// <summary> /// Adds a cache item to the database. The file is assumed to exist already. If that item exists /// already in the DB, it will be replaced with the new headers and statusCode, and the RC data will /// be updated. /// /// If the item did not exist and it does not fit in the cache, other items will be evicted. /// </summary> /// <param name="relFileName">The filename.</param> /// <param name="headers">The headers.</param> /// <param name="statusCode">The status code.</param> /// <param name="addToIndex">Whether a text/html file should be added to the Lucene index.</param> /// <param name="databaseContext">The database context.</param> /// <returns>True for success and false for failure.</returns> private bool AddCacheItemForExistingFile(string relFileName, NameValueCollection headers, short statusCode, bool addToIndex, RCDatabaseEntities databaseContext) { // If the headers do not contain "Content-Type", which should practically not happen, // (but servers are actually not required to send it) we set it to the default: // "application/octet-stream" if (headers["Content-Type"] == null) { headers["Content-Type"] = "application/octet-stream"; } long itemSize; // Get the cache and the file size try { itemSize = CacheItemFileSize(relFileName); } catch (Exception e) { _proxy.Logger.Warn("Could not compute file size.", e); return false; } GlobalCacheItem existingCacheItem = GetGlobalCacheItem(relFileName, databaseContext); // Look if we have to evict cache items first. long cacheOversize = _cacheSize + itemSize - _maxCacheSize; if (existingCacheItem != null) { cacheOversize -= -existingCacheItem.filesize; } if (cacheOversize > 0) { try { // We have to evict. We do until 5 % is free again. EvictCacheItems((long)(cacheOversize + CACHE_EVICTION_PERCENT * _maxCacheSize), databaseContext); // Always save after evicting, otherwise the CacheSize will not return the correct value // the next time and we will try evicting the same items again, which produces errors. databaseContext.SaveChanges(); } catch (Exception e) { _proxy.Logger.Warn("Could not evict cache items.", e); return false; } } if (existingCacheItem == null) { // Add database entry. try { AddCacheItemToDatabase(relFileName, headers, statusCode, itemSize, addToIndex, databaseContext); } catch (Exception e) { _proxy.Logger.Warn("Could not add cache item to the database.", e); return false; } } else { // Update database entry. try { UpdateCacheItemInDatabase(existingCacheItem, headers, statusCode, itemSize, databaseContext); } catch (Exception e) { _proxy.Logger.Warn("Could not add cache item to the database.", e); return false; } } return true; }
/// <summary> /// Checks whether an item is cached. Only the DB is being used, /// the disk contens are not ebing looked at. /// </summary> /// <param name="filename">The filename.</param> /// <param name="databaseContext">The database context.</param> /// <returns>If the item is cached.</returns> private bool IsCached(string filename, RCDatabaseEntities databaseContext) { if (filename.Length > 260) { return false; } return (from gci in databaseContext.GlobalCacheItem where gci.filename.Equals(filename) select 1).Count() != 0; }
/// <summary> /// Removes a cache item and deletes the file. This method is used internally, when evicting. /// </summary> /// <param name="cacheItem">The cache item.</param> /// <param name="databaseContext">The database context.</param> private void RemoveCacheItem(GlobalCacheItem cacheItem, RCDatabaseEntities databaseContext) { _proxy.Logger.Debug(String.Format("Removing from the cache: {0} Last request: {1}", cacheItem.filename, cacheItem.GlobalCacheRCData.lastRequestTime)); // Remove file Utils.DeleteFile(_cachePath + cacheItem.filename); // Remove cache item entry databaseContext.GlobalCacheItem.Remove(cacheItem); // If we're on the local proxy, we want to remove text documents from the Lucene index. if (_proxy is RCLocalProxy && GetHTTPMethodFromRelCacheFileName(cacheItem.filename).Equals("GET") && (cacheItem.responseHeaders.Contains("\"Content-Type\":[\"text/html") || cacheItem.responseHeaders.Contains("\"Content-Type\":[\"text/plain"))) { try { // remove the file from Lucene, if it is a GET text or HTML file. // We have made sure the content-type header is always present in the DB! ((RCLocalProxy)_proxy).IndexWrapper.DeleteDocument(FilePathToUri(cacheItem.filename)); } catch (Exception e) { _proxy.Logger.Warn("Could not remove document from the index.", e); } } }
/// <summary> /// Gets a new database context. This context must not be shared among threads! /// /// The context will never detect changes (which improves performance), as we have /// our own synchronization. /// </summary> /// <param name="autoDetectChangesEnabled">Whether this database context should detect changes to /// attached items and write them back when saveChanges is called. As we have our own synchronization, /// this should only be set to true, if we update an item. Otherwise performance will be bad.</param> /// <returns>A new database context.</returns> private RCDatabaseEntities GetNewDatabaseContext(bool autoDetectChangesEnabled) { // Create context and modify connection string to point to our DB file. RCDatabaseEntities result = new RCDatabaseEntities(); result.Database.Connection.ConnectionString = String.Format("data source=\"{0}\";Max Database Size={1};Max Buffer Size={2}", _proxy.ProxyPath + DATABASE_FILE_NAME, DATABASE_MAX_SIZE_MB, DATABASE_BUFFER_MAX_SIZE_KB); result.Configuration.AutoDetectChangesEnabled = autoDetectChangesEnabled; result.Configuration.ValidateOnSaveEnabled = false; return result; }
/// <summary> /// Gets the global cache RC data item for the specified HTTP method and URI, if it exists, /// and null otherwise. /// </summary> /// <param name="filename">The filename.</param> /// <param name="databaseContext">The database context.</param> /// <returns>The global cache RC data item or null.</returns> private GlobalCacheRCData GetGlobalCacheRCData(string filename, RCDatabaseEntities databaseContext) { if (filename.Length > 260) { return null; } return (from gcrc in databaseContext.GlobalCacheRCData where gcrc.filename.Equals(filename) select gcrc).FirstOrDefault(); }
/// <summary> /// Gets a global cache item and updates the rc metadata. /// /// databaseContext.SaveChanges() still needs to be called. /// </summary> /// <param name="filename">The filename.</param> /// <param name="databaseContext">The database context.</param> /// <returns>The item or null.</returns> private GlobalCacheItem GetGlobalCacheItemAsRequest(string filename, RCDatabaseEntities databaseContext) { GlobalCacheItem result = GetGlobalCacheItem(filename, databaseContext); if (result != null) { _proxy.Logger.Debug(String.Format("Updating request time and number of requests of {0}", filename)); // Modify the RC data result.GlobalCacheRCData.lastRequestTime = DateTime.Now; result.GlobalCacheRCData.numberOfRequests++; } return result; }
/// <summary> /// Evict items from the cache (also deleting the files from disk), until /// a total of bytesToEvict is deleted. /// /// LRU is the eviction strategy. /// </summary> /// <param name="bytesToEvict">The number of bytes to evict.</param> /// <param name="databaseContext">The database context.</param> private void EvictCacheItems(long bytesToEvict, RCDatabaseEntities databaseContext) { _proxy.Logger.Debug(String.Format("Evicting {0} bytes from the cache.", bytesToEvict)); long evicted = 0; IOrderedQueryable<GlobalCacheItem> orderedCacheItems = (from gci in databaseContext.GlobalCacheItem select gci). OrderBy(gci => gci.GlobalCacheRCData.lastRequestTime); foreach (GlobalCacheItem gci in orderedCacheItems) { evicted += gci.filesize; RemoveCacheItem(gci, databaseContext); if (evicted >= bytesToEvict) { break; } } }
/// <summary> /// Gets the number of cached items. There will be more GlobalCacheRCData items than this. /// </summary> /// <param name="databaseContext">The database context.</param> /// <returns>The number of cached items.</returns> private int CachedItems(RCDatabaseEntities databaseContext) { return (from gci in databaseContext.GlobalCacheItem select 1).Count(); }