/// <summary> /// Produces a list of all the files currently in the cache /// </summary> /// <returns>A list of all files in the cache</returns> private List <CacheFile> EnumerateAllCacheFiles() { IEnumerable <string> files = Directory.EnumerateFiles(this.root, "*.data", SearchOption.AllDirectories); List <CacheFile> fileData = new List <CacheFile>(); // Build a list of objects we can query foreach (string file in files) { Guid id; string hash = string.Empty; if (file.Contains(this.incoming)) { // This is in the incoming folder, skip it continue; } if (!CacheFile.ParseFileName(Path.GetFileName(file), out id, out hash)) { // Not a cache file, skip it continue; } CacheFile cacheFile = new CacheFile(); cacheFile.Load(this.root, id, hash); fileData.Add(cacheFile); } return(fileData); }
/// <summary> /// Gets the file size of a given file /// </summary> /// <param name="root">The root folder of the cache</param> /// <param name="id">The Id of the file to get the size of</param> /// <param name="hash">The has of the file</param> /// <returns>The file size in bytes</returns> public static ulong GetFileSizeBytes(string root, Guid id, string hash) { // TODO: Replace all string.Format with Path.Join FileInfo info = new FileInfo(CacheFile.GetFullFilePath(root, id, hash)); return((ulong)info.Length); }
/// <summary> /// Opens a file stream to the given asset /// </summary> /// <param name="id">The Id of the file</param> /// <param name="hash">The hash of the file</param> /// <returns>A read only file handle to the asset</returns> public FileStream GetReadFileStream(Guid id, string hash) { string path = Path.Combine(this.root, CacheFile.GetFolder(hash), CacheFile.GetFileName(id, hash)); File.SetLastAccessTime(path, DateTime.Now); return(File.OpenRead(path)); }
/// <summary> /// Walks through all assets in the file system and evicts any items that occur in the past /// to take the cache file system below the requested limit. /// This should only be called in a background thread since this is a time intensive operation. /// </summary> /// <param name="state">Ignored, object state</param> private void Evict(object state) { ulong cacheLimitBytes = (ulong)Settings.Default.MaxCacheSizeMB * 1048576; if (this.cacheSizeBytes < cacheLimitBytes) { // Cache isn't big enough to evict logger.Info( "Cache isn't large enough to require eviction. Max: {0} MB, Current: {1} MB", Settings.Default.MaxCacheSizeMB, this.cacheSizeBytes / 1048576); return; } logger.Warn( "Cache eviction is starting to prune. Max: {0} MB, Current: {1} MB", Settings.Default.MaxCacheSizeMB, this.cacheSizeBytes / 1048576); List <CacheFile> fileData = this.EnumerateAllCacheFiles(); // Run the list of elements and delete the files that were accessed the furthest in the past var filesSorted = from a in fileData orderby a.LastAccessed ascending select a; IEnumerator <CacheFile> fileEnumerator = filesSorted.GetEnumerator(); fileEnumerator.MoveNext(); // Delete files until we are within 90% of the limit while (this.cacheSizeBytes > (cacheLimitBytes * Settings.Default.CacheFreePercentage)) { CacheFile file = fileEnumerator.Current; lock (this.cacheSizeBytesLock) { this.cacheSizeBytes -= (ulong)file.Length; } logger.Warn( "Deleting last accessed {1} file {0}", file.LastAccessed, CacheFile.GetFileName(file.Id, file.Hash)); File.Delete(CacheFile.GetFullFilePath(this.root, file.Id, file.Hash)); if (!fileEnumerator.MoveNext()) { // There are no more files to clean break; } } logger.Warn( "Cache eviction is complete. Max: {0} MB, Current: {1} MB", Settings.Default.MaxCacheSizeMB, this.cacheSizeBytes / 1048576); this.evictingCache = false; }
/// <summary> /// Loads the details of the files /// </summary> /// <param name="root">The root folder of the cache</param> /// <param name="id">The id of the file</param> /// <param name="hash">The hash of the file</param> public void Load(string root, Guid id, string hash) { string path = CacheFile.GetFullFilePath(root, id, hash); FileInfo info = new FileInfo(path); this.Id = id; this.Hash = hash; this.LastAccessed = File.GetLastAccessTime(path); this.Length = (int)info.Length; }
/// <summary> /// Moves the file from the incoming temporary folder and moves it to permanent storage. /// The file is automatically marked as recently accessed in order to prevent it from being evicted. /// </summary> /// <param name="id">The Id of the file</param> /// <param name="hash">The hash of the file</param> public void CompleteFile(Guid id, string hash) { string fileName = CacheFile.GetFileName(id, hash); string src = Path.Combine(this.incoming, fileName); string dest = CacheFile.GetFullFilePath(this.root, id, hash); if (!Directory.Exists(Path.GetDirectoryName(dest))) { Directory.CreateDirectory(Path.GetDirectoryName(dest)); } // For some reason the cache server is asking to overwrite the file, if (CacheFile.IsFileCached(this.root, id, hash)) { File.Delete(CacheFile.GetFullFilePath(this.root, id, hash)); } File.Move(src, dest); File.SetLastAccessTime(dest, DateTime.Now); FileInfo info = new FileInfo(dest); lock (this.cacheSizeBytesLock) { // Increment the cache size by adding a file this.cacheSizeBytes += (ulong)info.Length; int limit = Settings.Default.MaxCacheSizeMB * 1048576; // Check we haven't exceeded the cap if (this.cacheSizeBytes > (ulong)limit && !this.evictingCache) { // We've exceeded the cache cap, request a cleanup this.evictingCache = true; ThreadPool.QueueUserWorkItem(new WaitCallback(this.Evict)); } } // Store a hit on the ojbect File.SetLastAccessTime(dest, DateTime.Now); logger.Info(CultureInfo.CurrentCulture, "Moving {0} to permanent cache", fileName); }
/// <summary> /// Processes the get command /// </summary> /// <param name="stream">The stream to the client</param> private void ProcessGet(NetworkStream stream) { // Read ID Guid id = UnityCacheUtilities.ReadGuid(stream); string hash = UnityCacheUtilities.ReadHash(stream); if (!CacheFile.IsFileCached(this.fileManager.Root, id, hash)) { logger.Info("GET: Cache miss. {0} {1}", id, hash); // File is not cached // Send command it's not cached byte[] code = new byte[1]; code[0] = 45; stream.Write(code, 0, 1); // Send id and hash UnityCacheUtilities.SendIdAndHashOnStream(stream, id, hash); } else { logger.Info("GET: Cache hit. {0} {1}", id, hash); using (MemoryStream memoryStream = new MemoryStream(49)) { // File is cached, send the response byte[] code = new byte[1]; code[0] = 43; memoryStream.Write(code, 0, 1); // Send the file size in bytes ulong bytesToBeWritten = CacheFile.GetFileSizeBytes(this.fileManager.Root, id, hash); // Dumb off by 1 hack byte[] fileSizeBytes = UnityCacheUtilities.GetUlongAsAsciiBytes(bytesToBeWritten); memoryStream.Write(fileSizeBytes, 0, fileSizeBytes.Length); // Send id and hash UnityCacheUtilities.SendIdAndHashOnStream(memoryStream, id, hash); // Send the file bytes FileStream fileStream = this.fileManager.GetReadFileStream(id, hash); byte[] buffer = new byte[this.streamBlockSize]; // Workaround to get enough bytes into a single packet so the Unity client doesn't choke byte[] header = memoryStream.GetBuffer(); stream.Write(header, 0, header.Length); while (bytesToBeWritten > 0) { int byteCount = (bytesToBeWritten > (ulong)this.streamBlockSize) ? this.streamBlockSize : (int)bytesToBeWritten; fileStream.Read(buffer, 0, byteCount); bytesToBeWritten -= (ulong)byteCount; stream.Write(buffer, 0, byteCount); } fileStream.Close(); } } // Notify listeners a get was processed if (this.OnGetProcessed != null) { this.OnGetProcessed(this, new EventArgs()); } }
/// <summary> /// Opens a file stream to the file that represents the temporary cache file. The caller should /// close the file and then call CompleteFile to move it to permanent storage. /// </summary> /// <param name="id">The id of the file</param> /// <param name="hash">The hash of the file</param> /// <returns>A file stream to the temporary file</returns> public FileStream GetTemporaryFile(Guid id, string hash) { string path = Path.Combine(this.incoming, CacheFile.GetFileName(id, hash)); return(File.OpenWrite(path)); }
/// <summary> /// Produces a list of all the files currently in the cache /// </summary> /// <returns>A list of all files in the cache</returns> private List<CacheFile> EnumerateAllCacheFiles() { IEnumerable<string> files = Directory.EnumerateFiles(this.root, "*.data", SearchOption.AllDirectories); List<CacheFile> fileData = new List<CacheFile>(); // Build a list of objects we can query foreach (string file in files) { Guid id; string hash = string.Empty; if (file.Contains(this.incoming)) { // This is in the incoming folder, skip it continue; } if (!CacheFile.ParseFileName(Path.GetFileName(file), out id, out hash)) { // Not a cache file, skip it continue; } CacheFile cacheFile = new CacheFile(); cacheFile.Load(this.root, id, hash); fileData.Add(cacheFile); } return fileData; }
/// <summary> /// Determines if the given file is cached or not /// </summary> /// <param name="root">The root folder of the cache</param> /// <param name="id">The Id of the file</param> /// <param name="hash">The hash of the file</param> /// <returns>True if the file is cached, false otherwise</returns> public static bool IsFileCached(string root, Guid id, string hash) { return(File.Exists(CacheFile.GetFullFilePath(root, id, hash))); }
/// <summary> /// Returns the full path of a given file /// </summary> /// <param name="root">The root folder of the cache</param> /// <param name="id">The Id of the file</param> /// <param name="hash">The hash of the file</param> /// <returns>The full path of the file</returns> public static string GetFullFilePath(string root, Guid id, string hash) { return(Path.Combine(root, CacheFile.GetFolder(hash), CacheFile.GetFileName(id, hash))); }