/// <summary>
/// Deletes an item from the cache.
/// </summary>
/// <remarks>
/// Note that our cache sync code (CheckForAndOrUploadMissingItem)
/// will fail if given an item to sync, and that item is subsequently
/// deleted from the local cache before the sync code gets around to
/// syncing it. This method is really only intended for cache
/// management purposes and not for general use. If that changes,
/// the cache sync code should change as well.
/// </remarks>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
public void DeleteItem(
ItemCacheContainer container,
string itemHash)
{
this.localCache.DeleteItem(container, itemHash);
this.cloudCache.DeleteItem(container, itemHash);
}
/// <summary>
/// Copies the given item from the cache to a new byte array.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A byte array containing a copy of the item.</returns>
public byte[] FetchItem(
ItemCacheContainer container,
string itemHash)
{
byte[] contents;
contents = this.localCache.FetchItem(container, itemHash);
if (contents == null)
{
contents = this.cloudCache.FetchItem(container, itemHash);
if (contents == null)
{
return null;
}
this.localCache.StoreItem(container, itemHash, contents);
}
else
{
// -
// Schedule cloud push on successful local read.
// REVIEW: Is this rare optimization really worth it?
// -
this.QueueItemForCloudSync(container, itemHash);
}
return contents;
}
/// <summary>
/// Gets a HashSet containing the hash keys of all the items in the
/// given container.
/// </summary>
/// <param name="container">Identifier for the cache container.</param>
/// <returns>A HashSet containing the hash keys.</returns>
public HashSet <string> GetItemsInContainer(ItemCacheContainer container)
{
// -
// REVIEW: What to return here? Both caches contents? Nothing?
// -
return(new HashSet <string>());
}
/// <summary>
/// Dumps the given item from the given cache and container.
/// </summary>
/// <param name="cache">Cache to look in.</param>
/// <param name="container">Container to look in.</param>
/// <param name="item">Item to dump.</param>
private static void DumpItem(IItemCache cache, ItemCacheContainer container, string item)
{
#if true
byte[] content = cache.FetchItem(container, item);
if (content != null)
{
BinaryWriter writer = new BinaryWriter(Console.OpenStandardOutput());
writer.Write(content);
writer.Close();
}
#endif
#if false
ResultSummaryRecord record = FetchRecord(cache, container, item);
if (record != null)
{
Console.WriteLine();
Console.WriteLine("IsVerificationTimeout = {0}", record.IsVerificationTimeout);
Console.WriteLine("IsFailure = {0}", record.IsFailure);
}
else
{
Console.WriteLine();
Console.WriteLine("FetchRecord failed for {0}", item);
}
#endif
}
/// <summary>
/// Copies the given item from the cache to a new byte array.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A byte array containing a copy of the item.</returns>
public byte[] FetchItem(
ItemCacheContainer container,
string itemHash)
{
byte[] contents;
contents = this.localCache.FetchItem(container, itemHash);
if (contents == null)
{
contents = this.cloudCache.FetchItem(container, itemHash);
if (contents == null)
{
return(null);
}
this.localCache.StoreItem(container, itemHash, contents);
}
else
{
// -
// Schedule cloud push on successful local read.
// REVIEW: Is this rare optimization really worth it?
// -
this.QueueItemForCloudSync(container, itemHash);
}
return(contents);
}
/// <summary>
/// Copies the given item from the cache to the given location in the
/// local file system.
/// </summary>
/// <remarks>
/// As with GetReadableStreamForItem, we first try locally and only
/// go to the cloud if needed.
/// </remarks>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <param name="localFilesystemDestinationPath">
/// Location in the local file system to copy the item.
/// </param>
public void FetchItemToFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemDestinationPath)
{
try
{
this.localCache.FetchItemToFile(container, itemHash, localFilesystemDestinationPath);
// -
// Schedule cloud push on successful local read.
// REVIEW: Is this rare optimization really worth it?
// -
this.QueueItemForCloudSync(container, itemHash);
}
catch (ObjectMissingFromCacheException)
{
// -
// If it is missing locally, try to retrieve it from the cloud.
// Note we stash a copy in the local cache prior to copying it
// to the desired local file.
// -
byte[] temp = this.cloudCache.FetchItem(container, itemHash);
if (temp == null)
{
throw new ObjectMissingFromCacheException(itemHash, "Item missing from multiplexed cache.");
}
this.localCache.StoreItem(container, itemHash, temp);
this.localCache.FetchItemToFile(container, itemHash, localFilesystemDestinationPath);
}
}
/// <summary>
/// Copies the given file from the local file system into the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="localFilesystemSourcePath">
/// Location in the local file system from which to source the item.
/// </param>
public void StoreItemFromFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemSourcePath)
{
this.localCache.StoreItemFromFile(container, itemHash, localFilesystemSourcePath);
this.QueueItemForCloudSync(container, itemHash);
}
/// <summary>
/// Deletes an item from the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
public void DeleteItem(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
cloudBlob.DeleteIfExists();
}
/// <summary>
/// Copies the given byte array to the desired cache item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="contents">Byte array containing the item.</param>
public void StoreItem(
ItemCacheContainer container,
string itemHash,
byte[] contents)
{
this.localCache.StoreItem(container, itemHash, contents);
this.QueueItemForCloudSync(container, itemHash);
}
/// <summary>
/// Checks whether the specified item exists in the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>
/// True if the specified item is in the cache, false otherwise.
/// </returns>
public bool ItemExists(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
return(cloudBlob.Exists());
}
/// <summary>
/// Copies the given byte array to the desired cache item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="contents">
/// Byte array containing the item.
/// </param>
public void StoreItem(
ItemCacheContainer container,
string itemHash,
byte[] contents)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
cloudBlob.UploadFromByteArray(contents, 0, contents.Length);
}
/// <summary>
/// Deletes an item from the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
public void DeleteItem(
ItemCacheContainer container,
string itemHash)
{
lock (this.cacheLock)
{
File.Delete(this.ItemPath(container, itemHash));
}
}
/// <summary>
/// Copies the given file from the local file system into the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="localFilesystemSourcePath">
/// Location in the local file system from which to source the item.
/// </param>
public void StoreItemFromFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemSourcePath)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
cloudBlob.UploadFromFile(localFilesystemSourcePath, FileMode.Open);
}
/// <summary>
/// Queue the given item for asynchronous cloud cache synchronization.
/// </summary>
/// <param name="container">
/// Identifier for the item's cache container.
/// </param>
/// <param name="itemHash">Hash key for the item.</param>
private void QueueItemForCloudSync(
ItemCacheContainer container,
string itemHash)
{
if (this.backgroundWorker != null)
{
this.backgroundWorker.QueueWork(this.CheckForAndOrUploadMissingItem, container, itemHash);
}
}
/// <summary>
/// Gets a HashSet containing the hash keys of all the items in the
/// given container.
/// </summary>
/// <param name="container">Identifier for the cache container.</param>
/// <returns>A HashSet containing the hash keys.</returns>
public HashSet <string> GetItemsInContainer(ItemCacheContainer container)
{
HashSet <string> itemHashes = new HashSet <string>();
foreach (CloudBlockBlob item in this.cloudContainers[(int)container].ListBlobs(null, true))
{
itemHashes.Add(item.Name);
}
return(itemHashes);
}
/// <summary>
/// Gets a HashSet containing the hash keys of all the items in the
/// given container.
/// </summary>
/// <param name="container">Identifier for the cache container.</param>
/// <returns>A HashSet containing the hash keys.</returns>
public HashSet <string> GetItemsInContainer(ItemCacheContainer container)
{
HashSet <string> itemHashes = new HashSet <string>();
foreach (string filename in Directory.EnumerateFiles(this.localPaths[(int)container]))
{
itemHashes.Add(Path.GetFileName(filename));
}
return(itemHashes);
}
/// <summary>
/// Initializes a new instance of the MultiplexerWrappedStream
/// class.
/// </summary>
/// <param name="stream">A stream to wrap.</param>
/// <param name="multiplexer">
/// The multiplexer cache instance owning item.
/// </param>
/// <param name="container">
/// The container for the item in the multiplexer cache.
/// </param>
/// <param name="itemHash">
/// The hash for the item in the multiplexer cache.
/// </param>
public MultiplexerWrappedStream(
Stream stream,
ItemCacheMultiplexer multiplexer,
ItemCacheContainer container,
string itemHash)
{
this.stream = stream;
this.multiplexer = multiplexer;
this.container = container;
this.itemHash = itemHash;
}
/// <summary>
/// Gets the size of the item.
/// Returns -1 if the item is absent.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>Size of the item in bytes, or -1 if item is absent.</returns>
public long GetItemSize(
ItemCacheContainer container,
string itemHash)
{
long size = this.localCache.GetItemSize(container, itemHash);
if (size == -1)
{
size = this.cloudCache.GetItemSize(container, itemHash);
}
return(size);
}
/// <summary>
/// Copies the given byte array to the desired cache item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="contents">
/// Byte array containing the item.
/// </param>
public void StoreItem(
ItemCacheContainer container,
string itemHash,
byte[] contents)
{
string itemPath = this.ItemPath(container, itemHash);
lock (this.cacheLock)
{
File.Delete(itemPath);
File.WriteAllBytes(itemPath, contents);
}
}
/// <summary>
/// Copies the given file from the local file system into the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="localFilesystemSourcePath">
/// Location in the local file system from which to source the item.
/// </param>
public void StoreItemFromFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemSourcePath)
{
string itemPath = this.ItemPath(container, itemHash);
lock (this.cacheLock)
{
File.Delete(itemPath);
File.Copy(localFilesystemSourcePath, itemPath);
}
}
/// <summary>
/// Gets the last-modified time of the item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A DateTimeOffset containing the item's last-modified time.</returns>
public DateTimeOffset?GetItemLastModifiedTime(
ItemCacheContainer container,
string itemHash)
{
DateTimeOffset?modifiedTime = this.localCache.GetItemLastModifiedTime(container, itemHash);
if (modifiedTime == null)
{
modifiedTime = this.cloudCache.GetItemLastModifiedTime(container, itemHash);
}
return(modifiedTime);
}
/// <summary>
/// Gets the last-modified time of the item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A DateTimeOffset containing the item's last-modified time.</returns>
public DateTimeOffset?GetItemLastModifiedTime(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
if (cloudBlob.Exists())
{
return(cloudBlob.Properties.LastModified);
}
return(null);
}
/// <summary>
/// Gets the size of the item.
/// Returns -1 if the item is absent.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>Size of the item in bytes, or -1 if item is absent.</returns>
public long GetItemSize(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
if (cloudBlob.Exists())
{
return(cloudBlob.Properties.Length);
}
return(-1);
}
/// <summary>
/// Gets the last-modified time of the item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A DateTimeOffset containing the item's last-modified time.</returns>
public DateTimeOffset?GetItemLastModifiedTime(
ItemCacheContainer container,
string itemHash)
{
lock (this.cacheLock)
{
FileInfo fileInfo = new FileInfo(this.ItemPath(container, itemHash));
if (fileInfo.Exists)
{
return(fileInfo.CreationTimeUtc);
}
return(null);
}
}
/// <summary>
/// Gets the size of the item.
/// Returns -1 if the item is absent.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>Size of the item in bytes, or -1 if item is absent.</returns>
public long GetItemSize(
ItemCacheContainer container,
string itemHash)
{
lock (this.cacheLock)
{
FileInfo fileInfo = new FileInfo(this.ItemPath(container, itemHash));
if (fileInfo.Exists)
{
return(fileInfo.Length);
}
return(-1);
}
}
/// <summary>
/// Copies the given item from the cache to a new byte array.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A byte array containing a copy of the item.</returns>
public byte[] FetchItem(
ItemCacheContainer container,
string itemHash)
{
string itemPath = this.ItemPath(container, itemHash);
lock (this.cacheLock)
{
if (!File.Exists(itemPath))
{
return null;
}
return File.ReadAllBytes(itemPath);
}
}
/// <summary>
/// Copies the given item from the cache to a new byte array.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A byte array containing a copy of the item.</returns>
public byte[] FetchItem(
ItemCacheContainer container,
string itemHash)
{
string itemPath = this.ItemPath(container, itemHash);
lock (this.cacheLock)
{
if (!File.Exists(itemPath))
{
return(null);
}
return(File.ReadAllBytes(itemPath));
}
}
/// <summary>
/// Copies the given item from the cache to the given location in the
/// local file system.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <param name="localFilesystemDestinationPath">
/// Location in the local file system to copy the item.
/// </param>
public void FetchItemToFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemDestinationPath)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
try
{
cloudBlob.DownloadToFile(localFilesystemDestinationPath, FileMode.Create);
}
catch (Microsoft.WindowsAzure.Storage.StorageException)
{
throw new ObjectMissingFromCacheException(itemHash, "Item missing from cloud cache.");
}
}
/// <summary>
/// Copies the given item from the cache to a new byte array.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A byte array containing a copy of the item.</returns>
public byte[] FetchItem(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
if (!cloudBlob.Exists())
{
return(null);
}
using (MemoryStream memoryStream = new MemoryStream())
{
cloudBlob.DownloadToStream(memoryStream);
return(memoryStream.ToArray());
}
}
/// <summary>
/// Copies the given item from the cache to the given location in the
/// local file system.
/// </summary>
/// <remarks>
/// This method is a performance optimization over getting a readable
/// stream for the item and copying it to a local file using CopyTo().
/// </remarks>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <param name="localFilesystemDestinationPath">
/// Location in the local file system to copy the item.
/// </param>
public void FetchItemToFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemDestinationPath)
{
lock (this.cacheLock)
{
try
{
Directory.CreateDirectory(Path.GetDirectoryName(localFilesystemDestinationPath));
File.Copy(this.ItemPath(container, itemHash), localFilesystemDestinationPath, true);
}
catch (FileNotFoundException)
{
throw new ObjectMissingFromCacheException(itemHash, "Item missing from local cache.");
}
}
}
/// <summary>
/// Retrieves the requested result from the given cache.
/// </summary>
/// <param name="cache">Cache to query.</param>
/// <param name="container">Container to query.</param>
/// <param name="itemHash">Result to get.</param>
/// <returns>The requested ResultSummaryRecord, or null if not found.</returns>
private static ResultSummaryRecord FetchRecord(IItemCache cache, ItemCacheContainer container, string itemHash)
{
byte[] result = cache.FetchItem(container, itemHash);
if (result != null)
{
MemoryStream resultStream = new MemoryStream(result);
try
{
using (StreamReader inReader = new StreamReader(resultStream))
{
string xmlSummary = inReader.ReadToEnd();
ResultSummaryRecord record = ResultSummaryRecord.FromXml(xmlSummary);
if (record == null)
{
Console.WriteLine("FromXml failed for {0}", itemHash);
}
return(record);
}
}
catch (System.Xml.XmlException ex)
{
Console.WriteLine("Malformed xml in {0}: {1}", itemHash, ex.ToString());
return(null);
}
finally
{
resultStream.Dispose();
}
}
else
{
Console.WriteLine("FetchItem failed for {0}", itemHash);
return(null);
}
}
/// <summary>
/// Check if the given item is already present in the cloud cache,
/// and if not, upload the local cache item to the cloud.
/// </summary>
/// <param name="containerObject">
/// Identifier for the item's cache container.
/// </param>
/// <param name="itemHashObject">Hash key for the item.</param>
private void CheckForAndOrUploadMissingItem(
object containerObject,
object itemHashObject)
{
ItemCacheContainer container = (ItemCacheContainer)containerObject;
string itemHash = (string)itemHashObject;
if (this.localCache.GetItemSize(container, itemHash) > MaxUploadSizeThreshold)
{
Logger.WriteLine(string.Format(
"Warning: skipping upload of {0} because it's really big. Compress?",
itemHash));
return;
}
// -
// Check if the item is already present in the cloud cache.
// TODO present doesn't mean we don't want to overwrite it (eg when
// replacing a Failed verification result with a succeeding one.)
// -
if (this.cloudCache.ItemExists(container, itemHash))
{
return;
}
// -
// The item is missing from the cloud cache. Upload it.
// -
byte[] temp = this.localCache.FetchItem(container, itemHash);
if (temp == null)
{
// This should never happen barring a serious logic error.
throw new ObjectMissingFromCacheException(itemHash, "Can't upload non-existant cache item!");
}
this.cloudCache.StoreItem(container, itemHash, temp);
}
/// <summary>
/// Gets the last-modified time of the item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A DateTimeOffset containing the item's last-modified time.</returns>
public DateTimeOffset? GetItemLastModifiedTime(
ItemCacheContainer container,
string itemHash)
{
DateTimeOffset? modifiedTime = this.localCache.GetItemLastModifiedTime(container, itemHash);
if (modifiedTime == null)
{
modifiedTime = this.cloudCache.GetItemLastModifiedTime(container, itemHash);
}
return modifiedTime;
}
/// <summary>
/// Copies the given file from the local file system into the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="localFilesystemSourcePath">
/// Location in the local file system from which to source the item.
/// </param>
public void StoreItemFromFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemSourcePath)
{
this.localCache.StoreItemFromFile(container, itemHash, localFilesystemSourcePath);
this.QueueItemForCloudSync(container, itemHash);
}
/// <summary>
/// Copies the given item from the cache to a new byte array.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A byte array containing a copy of the item.</returns>
public byte[] FetchItem(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
if (!cloudBlob.Exists())
{
return null;
}
using (MemoryStream memoryStream = new MemoryStream())
{
cloudBlob.DownloadToStream(memoryStream);
return memoryStream.ToArray();
}
}
/// <summary>
/// Checks whether the specified item exists in the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>
/// True if the specified item is in the cache, false otherwise.
/// </returns>
public bool ItemExists(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
return cloudBlob.Exists();
}
/// <summary>
/// Gets the last-modified time of the item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A DateTimeOffset containing the item's last-modified time.</returns>
public DateTimeOffset? GetItemLastModifiedTime(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
if (cloudBlob.Exists())
{
return cloudBlob.Properties.LastModified;
}
return null;
}
/// <summary>
/// Gets the size of the item.
/// Returns -1 if the item is absent.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>Size of the item in bytes, or -1 if item is absent.</returns>
public long GetItemSize(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
if (cloudBlob.Exists())
{
return cloudBlob.Properties.Length;
}
return -1;
}
/// <summary>
/// Initializes a new instance of the MultiplexerWrappedStream
/// class.
/// </summary>
/// <param name="stream">A stream to wrap.</param>
/// <param name="multiplexer">
/// The multiplexer cache instance owning item.
/// </param>
/// <param name="container">
/// The container for the item in the multiplexer cache.
/// </param>
/// <param name="itemHash">
/// The hash for the item in the multiplexer cache.
/// </param>
public MultiplexerWrappedStream(
Stream stream,
ItemCacheMultiplexer multiplexer,
ItemCacheContainer container,
string itemHash)
{
this.stream = stream;
this.multiplexer = multiplexer;
this.container = container;
this.itemHash = itemHash;
}
/// <summary>
/// Deletes an item from the cache.
/// </summary>
/// <remarks>
/// Note that our cache sync code (CheckForAndOrUploadMissingItem)
/// will fail if given an item to sync, and that item is subsequently
/// deleted from the local cache before the sync code gets around to
/// syncing it. This method is really only intended for cache
/// management purposes and not for general use. If that changes,
/// the cache sync code should change as well.
/// </remarks>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
public void DeleteItem(
ItemCacheContainer container,
string itemHash)
{
this.localCache.DeleteItem(container, itemHash);
this.cloudCache.DeleteItem(container, itemHash);
}
/// <summary>
/// Public API for private CheckForAndOrUploadMissingItem method.
/// </summary>
/// <param name="container">
/// Identifier for the item's cache container.
/// </param>
/// <param name="itemHash">Hash key for the item.</param>
public void SyncItemToCloud(
ItemCacheContainer container,
string itemHash)
{
this.CheckForAndOrUploadMissingItem(container, itemHash);
}
/// <summary>
/// Gets the size of the item.
/// Returns -1 if the item is absent.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>Size of the item in bytes, or -1 if item is absent.</returns>
public long GetItemSize(
ItemCacheContainer container,
string itemHash)
{
long size = this.localCache.GetItemSize(container, itemHash);
if (size == -1)
{
size = this.cloudCache.GetItemSize(container, itemHash);
}
return size;
}
/// <summary>
/// Dumps the given item from the given cache and container.
/// </summary>
/// <param name="cache">Cache to look in.</param>
/// <param name="container">Container to look in.</param>
/// <param name="item">Item to dump.</param>
private static void DumpItem(IItemCache cache, ItemCacheContainer container, string item)
{
#if true
byte[] content = cache.FetchItem(container, item);
if (content != null)
{
BinaryWriter writer = new BinaryWriter(Console.OpenStandardOutput());
writer.Write(content);
writer.Close();
}
#endif
#if false
ResultSummaryRecord record = FetchRecord(cache, container, item);
if (record != null)
{
Console.WriteLine();
Console.WriteLine("IsVerificationTimeout = {0}", record.IsVerificationTimeout);
Console.WriteLine("IsFailure = {0}", record.IsFailure);
}
else
{
Console.WriteLine();
Console.WriteLine("FetchRecord failed for {0}", item);
}
#endif
}
/// <summary>
/// Copies the given item from the cache to the given location in the
/// local file system.
/// </summary>
/// <remarks>
/// As with GetReadableStreamForItem, we first try locally and only
/// go to the cloud if needed.
/// </remarks>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <param name="localFilesystemDestinationPath">
/// Location in the local file system to copy the item.
/// </param>
public void FetchItemToFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemDestinationPath)
{
try
{
this.localCache.FetchItemToFile(container, itemHash, localFilesystemDestinationPath);
// -
// Schedule cloud push on successful local read.
// REVIEW: Is this rare optimization really worth it?
// -
this.QueueItemForCloudSync(container, itemHash);
}
catch (ObjectMissingFromCacheException)
{
// -
// If it is missing locally, try to retrieve it from the cloud.
// Note we stash a copy in the local cache prior to copying it
// to the desired local file.
// -
byte[] temp = this.cloudCache.FetchItem(container, itemHash);
if (temp == null)
{
throw new ObjectMissingFromCacheException(itemHash, "Item missing from multiplexed cache.");
}
this.localCache.StoreItem(container, itemHash, temp);
this.localCache.FetchItemToFile(container, itemHash, localFilesystemDestinationPath);
}
}
/// <summary>
/// Copies the given byte array to the desired cache item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="contents">Byte array containing the item.</param>
public void StoreItem(
ItemCacheContainer container,
string itemHash,
byte[] contents)
{
this.localCache.StoreItem(container, itemHash, contents);
this.QueueItemForCloudSync(container, itemHash);
}
/// <summary>
/// Gets the local path name for the given item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>Path for the item.</returns>
private string ItemPath(ItemCacheContainer container, string itemHash)
{
return Path.Combine(
this.localPaths[(int)container],
itemHash);
}
/// <summary>
/// Compares the contents of two caches.
/// </summary>
/// <param name="queriedCaches">Caches to compare.</param>
/// <param name="queriedContainers">Containers in those caches to compare.</param>
private static void CompareCacheContainers(
IItemCache[] queriedCaches,
ItemCacheContainer[] queriedContainers)
{
foreach (ItemCacheContainer container in queriedContainers)
{
IItemCache cacheA = queriedCaches[0];
IItemCache cacheB = queriedCaches[1];
HashSet<string> cacheAItems = cacheA.GetItemsInContainer(container);
HashSet<string> cacheBItems = cacheB.GetItemsInContainer(container);
Console.WriteLine("There are {0} items in the {1} cache {2} container.", cacheAItems.Count, cacheA.Name, container.ToString());
Console.WriteLine("There are {0} items in the {1} cache {2} container.", cacheBItems.Count, cacheB.Name, container.ToString());
HashSet<string> syncedItems = new HashSet<string>(cacheAItems);
syncedItems.IntersectWith(cacheBItems);
Console.WriteLine("There are {0} items present in both cache's {1} container.", syncedItems.Count, container);
Console.WriteLine();
}
}
/// <summary>
/// Deletes the items from the given caches and containers
/// that have an earlier last modified time than the given one.
/// </summary>
/// <param name="queriedCaches">Caches to look in.</param>
/// <param name="queriedContainers">Containers to look in.</param>
/// <param name="queriedDate">Date to compare against.</param>
private static void DeleteItems(
IItemCache[] queriedCaches,
ItemCacheContainer[] queriedContainers,
DateTimeOffset queriedDate)
{
if (!DeleteItemsConfirmation())
{
return;
}
foreach (IItemCache cache in queriedCaches)
{
foreach (ItemCacheContainer container in queriedContainers)
{
HashSet<string> items = cache.GetItemsInContainer(container);
foreach (string item in items)
{
DateTimeOffset? lastModified = cache.GetItemLastModifiedTime(container, item);
if (lastModified.HasValue && (lastModified.Value.CompareTo(queriedDate) < 0))
{
cache.DeleteItem(container, item);
}
}
}
}
}
/// <summary>
/// Copies the given item from the cache to the given location in the
/// local file system.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <param name="localFilesystemDestinationPath">
/// Location in the local file system to copy the item.
/// </param>
public void FetchItemToFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemDestinationPath)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
try
{
cloudBlob.DownloadToFile(localFilesystemDestinationPath, FileMode.Create);
}
catch (Microsoft.WindowsAzure.Storage.StorageException)
{
throw new ObjectMissingFromCacheException(itemHash, "Item missing from cloud cache.");
}
}
/// <summary>
/// Gets a HashSet containing the hash keys of all the items in the
/// given container.
/// </summary>
/// <param name="container">Identifier for the cache container.</param>
/// <returns>A HashSet containing the hash keys.</returns>
public HashSet<string> GetItemsInContainer(ItemCacheContainer container)
{
// -
// REVIEW: What to return here? Both caches contents? Nothing?
// -
return new HashSet<string>();
}
/// <summary>
/// Copies the given byte array to the desired cache item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="contents">
/// Byte array containing the item.
/// </param>
public void StoreItem(
ItemCacheContainer container,
string itemHash,
byte[] contents)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
cloudBlob.UploadFromByteArray(contents, 0, contents.Length);
}
/// <summary>
/// Copies the given file from the local file system into the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container to hold the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the item.
/// </param>
/// <param name="localFilesystemSourcePath">
/// Location in the local file system from which to source the item.
/// </param>
public void StoreItemFromFile(
ItemCacheContainer container,
string itemHash,
string localFilesystemSourcePath)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
cloudBlob.UploadFromFile(localFilesystemSourcePath, FileMode.Open);
}
/// <summary>
/// Provides a status report of the number of items in the specified
/// cache containers, and optionally a list of those items.
/// </summary>
/// <param name="queriedCaches">Caches to examine.</param>
/// <param name="queriedContainers">Containers in those caches to examine.</param>
/// <param name="listContents">Whether to list the cache contents.</param>
private static void CacheStatus(
IItemCache[] queriedCaches,
ItemCacheContainer[] queriedContainers,
bool listContents)
{
string lineTerminator = ".";
if (listContents)
{
lineTerminator = ":";
}
foreach (IItemCache cache in queriedCaches)
{
foreach (ItemCacheContainer container in queriedContainers)
{
HashSet<string> items = cache.GetItemsInContainer(container);
Console.WriteLine("{0} cache {1} container holds {2} items{3}", cache.Name, container.ToString(), items.Count, lineTerminator);
if (listContents)
{
foreach (string item in items)
{
////Console.WriteLine("Item: {0}, Date:{1}", item, cache.GetItemLastModifiedTime(container, item));
Console.WriteLine(item);
}
Console.WriteLine();
}
}
Console.WriteLine();
}
}
/// <summary>
/// Deletes an item from the cache.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
public void DeleteItem(
ItemCacheContainer container,
string itemHash)
{
CloudBlockBlob cloudBlob = this.cloudContainers[(int)container].GetBlockBlobReference(itemHash);
cloudBlob.DeleteIfExists();
}
/// <summary>
/// Deletes the given items from the given caches and containers.
/// </summary>
/// <param name="queriedCaches">Caches to look in.</param>
/// <param name="queriedContainers">Containers to look in.</param>
/// <param name="queriedItems">Items to delete.</param>
private static void DeleteItems(
IItemCache[] queriedCaches,
ItemCacheContainer[] queriedContainers,
string queriedItems)
{
if (queriedItems == "*")
{
if (!DeleteItemsConfirmation())
{
return;
}
}
foreach (IItemCache cache in queriedCaches)
{
foreach (ItemCacheContainer container in queriedContainers)
{
if (queriedItems == "*")
{
HashSet<string> items = cache.GetItemsInContainer(container);
foreach (string item in items)
{
cache.DeleteItem(container, item);
}
}
else
{
cache.DeleteItem(container, queriedItems);
}
}
}
}
/// <summary>
/// Gets a HashSet containing the hash keys of all the items in the
/// given container.
/// </summary>
/// <param name="container">Identifier for the cache container.</param>
/// <returns>A HashSet containing the hash keys.</returns>
public HashSet<string> GetItemsInContainer(ItemCacheContainer container)
{
HashSet<string> itemHashes = new HashSet<string>();
foreach (CloudBlockBlob item in this.cloudContainers[(int)container].ListBlobs(null, true))
{
itemHashes.Add(item.Name);
}
return itemHashes;
}
/// <summary>
/// Dumps the given items from the given caches and containers.
/// </summary>
/// <param name="queriedCaches">Caches to look in.</param>
/// <param name="queriedContainers">Containers to look in.</param>
/// <param name="queriedItems">Items to dump.</param>
private static void DumpItems(
IItemCache[] queriedCaches,
ItemCacheContainer[] queriedContainers,
string queriedItems)
{
foreach (IItemCache cache in queriedCaches)
{
foreach (ItemCacheContainer container in queriedContainers)
{
if (queriedItems == "*")
{
HashSet<string> items = cache.GetItemsInContainer(container);
foreach (string item in items)
{
DumpItem(cache, container, item);
}
}
else
{
DumpItem(cache, container, queriedItems);
}
}
}
}
/// <summary>
/// Queue the given item for asynchronous cloud cache synchronization.
/// </summary>
/// <param name="container">
/// Identifier for the item's cache container.
/// </param>
/// <param name="itemHash">Hash key for the item.</param>
private void QueueItemForCloudSync(
ItemCacheContainer container,
string itemHash)
{
if (this.backgroundWorker != null)
{
this.backgroundWorker.QueueWork(this.CheckForAndOrUploadMissingItem, container, itemHash);
}
}
/// <summary>
/// Retrieves the requested result from the given cache.
/// </summary>
/// <param name="cache">Cache to query.</param>
/// <param name="container">Container to query.</param>
/// <param name="itemHash">Result to get.</param>
/// <returns>The requested ResultSummaryRecord, or null if not found.</returns>
private static ResultSummaryRecord FetchRecord(IItemCache cache, ItemCacheContainer container, string itemHash)
{
byte[] result = cache.FetchItem(container, itemHash);
if (result != null)
{
MemoryStream resultStream = new MemoryStream(result);
try
{
using (StreamReader inReader = new StreamReader(resultStream))
{
string xmlSummary = inReader.ReadToEnd();
ResultSummaryRecord record = ResultSummaryRecord.FromXml(xmlSummary);
if (record == null)
{
Console.WriteLine("FromXml failed for {0}", itemHash);
}
return record;
}
}
catch (System.Xml.XmlException ex)
{
Console.WriteLine("Malformed xml in {0}: {1}", itemHash, ex.ToString());
return null;
}
finally
{
resultStream.Dispose();
}
}
else
{
Console.WriteLine("FetchItem failed for {0}", itemHash);
return null;
}
}
/// <summary>
/// Gets the last-modified time of the item.
/// </summary>
/// <param name="container">
/// Identifier for the cache container holding the item.
/// </param>
/// <param name="itemHash">
/// Hash key for the desired item.
/// </param>
/// <returns>A DateTimeOffset containing the item's last-modified time.</returns>
public DateTimeOffset? GetItemLastModifiedTime(
ItemCacheContainer container,
string itemHash)
{
lock (this.cacheLock)
{
FileInfo fileInfo = new FileInfo(this.ItemPath(container, itemHash));
if (fileInfo.Exists)
{
return fileInfo.CreationTimeUtc;
}
return null;
}
}
