/// <summary>
/// Produces a list of all the files currently in the cache
/// </summary>
/// <returns>A list of all files in the cache</returns>
private List <CacheFile> EnumerateAllCacheFiles()
{
IEnumerable <string> files = Directory.EnumerateFiles(this.root, "*.data", SearchOption.AllDirectories);
List <CacheFile> fileData = new List <CacheFile>();
// Build a list of objects we can query
foreach (string file in files)
{
Guid id;
string hash = string.Empty;
if (file.Contains(this.incoming))
{
// This is in the incoming folder, skip it
continue;
}
if (!CacheFile.ParseFileName(Path.GetFileName(file), out id, out hash))
{
// Not a cache file, skip it
continue;
}
CacheFile cacheFile = new CacheFile();
cacheFile.Load(this.root, id, hash);
fileData.Add(cacheFile);
}
return(fileData);
}
/// <summary>
/// Gets the file size of a given file
/// </summary>
/// <param name="root">The root folder of the cache</param>
/// <param name="id">The Id of the file to get the size of</param>
/// <param name="hash">The has of the file</param>
/// <returns>The file size in bytes</returns>
public static ulong GetFileSizeBytes(string root, Guid id, string hash)
{
// TODO: Replace all string.Format with Path.Join
FileInfo info = new FileInfo(CacheFile.GetFullFilePath(root, id, hash));
return((ulong)info.Length);
}
/// <summary>
/// Opens a file stream to the given asset
/// </summary>
/// <param name="id">The Id of the file</param>
/// <param name="hash">The hash of the file</param>
/// <returns>A read only file handle to the asset</returns>
public FileStream GetReadFileStream(Guid id, string hash)
{
string path = Path.Combine(this.root, CacheFile.GetFolder(hash), CacheFile.GetFileName(id, hash));
File.SetLastAccessTime(path, DateTime.Now);
return(File.OpenRead(path));
}
/// <summary>
/// Walks through all assets in the file system and evicts any items that occur in the past
/// to take the cache file system below the requested limit.
/// This should only be called in a background thread since this is a time intensive operation.
/// </summary>
/// <param name="state">Ignored, object state</param>
private void Evict(object state)
{
ulong cacheLimitBytes = (ulong)Settings.Default.MaxCacheSizeMB * 1048576;
if (this.cacheSizeBytes < cacheLimitBytes)
{
// Cache isn't big enough to evict
logger.Info(
"Cache isn't large enough to require eviction. Max: {0} MB, Current: {1} MB",
Settings.Default.MaxCacheSizeMB,
this.cacheSizeBytes / 1048576);
return;
}
logger.Warn(
"Cache eviction is starting to prune. Max: {0} MB, Current: {1} MB",
Settings.Default.MaxCacheSizeMB,
this.cacheSizeBytes / 1048576);
List <CacheFile> fileData = this.EnumerateAllCacheFiles();
// Run the list of elements and delete the files that were accessed the furthest in the past
var filesSorted = from a in fileData
orderby a.LastAccessed ascending
select a;
IEnumerator <CacheFile> fileEnumerator = filesSorted.GetEnumerator();
fileEnumerator.MoveNext();
// Delete files until we are within 90% of the limit
while (this.cacheSizeBytes > (cacheLimitBytes * Settings.Default.CacheFreePercentage))
{
CacheFile file = fileEnumerator.Current;
lock (this.cacheSizeBytesLock)
{
this.cacheSizeBytes -= (ulong)file.Length;
}
logger.Warn(
"Deleting last accessed {1} file {0}",
file.LastAccessed,
CacheFile.GetFileName(file.Id, file.Hash));
File.Delete(CacheFile.GetFullFilePath(this.root, file.Id, file.Hash));
if (!fileEnumerator.MoveNext())
{
// There are no more files to clean
break;
}
}
logger.Warn(
"Cache eviction is complete. Max: {0} MB, Current: {1} MB",
Settings.Default.MaxCacheSizeMB,
this.cacheSizeBytes / 1048576);
this.evictingCache = false;
}
/// <summary>
/// Loads the details of the files
/// </summary>
/// <param name="root">The root folder of the cache</param>
/// <param name="id">The id of the file</param>
/// <param name="hash">The hash of the file</param>
public void Load(string root, Guid id, string hash)
{
string path = CacheFile.GetFullFilePath(root, id, hash);
FileInfo info = new FileInfo(path);
this.Id = id;
this.Hash = hash;
this.LastAccessed = File.GetLastAccessTime(path);
this.Length = (int)info.Length;
}
/// <summary>
/// Moves the file from the incoming temporary folder and moves it to permanent storage.
/// The file is automatically marked as recently accessed in order to prevent it from being evicted.
/// </summary>
/// <param name="id">The Id of the file</param>
/// <param name="hash">The hash of the file</param>
public void CompleteFile(Guid id, string hash)
{
string fileName = CacheFile.GetFileName(id, hash);
string src = Path.Combine(this.incoming, fileName);
string dest = CacheFile.GetFullFilePath(this.root, id, hash);
if (!Directory.Exists(Path.GetDirectoryName(dest)))
{
Directory.CreateDirectory(Path.GetDirectoryName(dest));
}
// For some reason the cache server is asking to overwrite the file,
if (CacheFile.IsFileCached(this.root, id, hash))
{
File.Delete(CacheFile.GetFullFilePath(this.root, id, hash));
}
File.Move(src, dest);
File.SetLastAccessTime(dest, DateTime.Now);
FileInfo info = new FileInfo(dest);
lock (this.cacheSizeBytesLock)
{
// Increment the cache size by adding a file
this.cacheSizeBytes += (ulong)info.Length;
int limit = Settings.Default.MaxCacheSizeMB * 1048576;
// Check we haven't exceeded the cap
if (this.cacheSizeBytes > (ulong)limit && !this.evictingCache)
{
// We've exceeded the cache cap, request a cleanup
this.evictingCache = true;
ThreadPool.QueueUserWorkItem(new WaitCallback(this.Evict));
}
}
// Store a hit on the ojbect
File.SetLastAccessTime(dest, DateTime.Now);
logger.Info(CultureInfo.CurrentCulture, "Moving {0} to permanent cache", fileName);
}
/// <summary>
/// Processes the get command
/// </summary>
/// <param name="stream">The stream to the client</param>
private void ProcessGet(NetworkStream stream)
{
// Read ID
Guid id = UnityCacheUtilities.ReadGuid(stream);
string hash = UnityCacheUtilities.ReadHash(stream);
if (!CacheFile.IsFileCached(this.fileManager.Root, id, hash))
{
logger.Info("GET: Cache miss. {0} {1}", id, hash);
// File is not cached
// Send command it's not cached
byte[] code = new byte[1];
code[0] = 45;
stream.Write(code, 0, 1);
// Send id and hash
UnityCacheUtilities.SendIdAndHashOnStream(stream, id, hash);
}
else
{
logger.Info("GET: Cache hit. {0} {1}", id, hash);
using (MemoryStream memoryStream = new MemoryStream(49))
{
// File is cached, send the response
byte[] code = new byte[1];
code[0] = 43;
memoryStream.Write(code, 0, 1);
// Send the file size in bytes
ulong bytesToBeWritten = CacheFile.GetFileSizeBytes(this.fileManager.Root, id, hash); // Dumb off by 1 hack
byte[] fileSizeBytes = UnityCacheUtilities.GetUlongAsAsciiBytes(bytesToBeWritten);
memoryStream.Write(fileSizeBytes, 0, fileSizeBytes.Length);
// Send id and hash
UnityCacheUtilities.SendIdAndHashOnStream(memoryStream, id, hash);
// Send the file bytes
FileStream fileStream = this.fileManager.GetReadFileStream(id, hash);
byte[] buffer = new byte[this.streamBlockSize];
// Workaround to get enough bytes into a single packet so the Unity client doesn't choke
byte[] header = memoryStream.GetBuffer();
stream.Write(header, 0, header.Length);
while (bytesToBeWritten > 0)
{
int byteCount = (bytesToBeWritten > (ulong)this.streamBlockSize) ? this.streamBlockSize : (int)bytesToBeWritten;
fileStream.Read(buffer, 0, byteCount);
bytesToBeWritten -= (ulong)byteCount;
stream.Write(buffer, 0, byteCount);
}
fileStream.Close();
}
}
// Notify listeners a get was processed
if (this.OnGetProcessed != null)
{
this.OnGetProcessed(this, new EventArgs());
}
}
/// <summary>
/// Opens a file stream to the file that represents the temporary cache file. The caller should
/// close the file and then call CompleteFile to move it to permanent storage.
/// </summary>
/// <param name="id">The id of the file</param>
/// <param name="hash">The hash of the file</param>
/// <returns>A file stream to the temporary file</returns>
public FileStream GetTemporaryFile(Guid id, string hash)
{
string path = Path.Combine(this.incoming, CacheFile.GetFileName(id, hash));
return(File.OpenWrite(path));
}
/// <summary>
/// Produces a list of all the files currently in the cache
/// </summary>
/// <returns>A list of all files in the cache</returns>
private List<CacheFile> EnumerateAllCacheFiles()
{
IEnumerable<string> files = Directory.EnumerateFiles(this.root, "*.data", SearchOption.AllDirectories);
List<CacheFile> fileData = new List<CacheFile>();
// Build a list of objects we can query
foreach (string file in files)
{
Guid id;
string hash = string.Empty;
if (file.Contains(this.incoming))
{
// This is in the incoming folder, skip it
continue;
}
if (!CacheFile.ParseFileName(Path.GetFileName(file), out id, out hash))
{
// Not a cache file, skip it
continue;
}
CacheFile cacheFile = new CacheFile();
cacheFile.Load(this.root, id, hash);
fileData.Add(cacheFile);
}
return fileData;
}
/// <summary>
/// Determines if the given file is cached or not
/// </summary>
/// <param name="root">The root folder of the cache</param>
/// <param name="id">The Id of the file</param>
/// <param name="hash">The hash of the file</param>
/// <returns>True if the file is cached, false otherwise</returns>
public static bool IsFileCached(string root, Guid id, string hash)
{
return(File.Exists(CacheFile.GetFullFilePath(root, id, hash)));
}
/// <summary>
/// Returns the full path of a given file
/// </summary>
/// <param name="root">The root folder of the cache</param>
/// <param name="id">The Id of the file</param>
/// <param name="hash">The hash of the file</param>
/// <returns>The full path of the file</returns>
public static string GetFullFilePath(string root, Guid id, string hash)
{
return(Path.Combine(root, CacheFile.GetFolder(hash), CacheFile.GetFileName(id, hash)));
}
