// Called by AzureBlobSyncProvider.UpdateItem to help with writing item updates
internal SyncedBlobAttributes UpdateFile(
string oldName,
FileData fileData,
string relativePath,
Stream dataStream,
DateTime expectedLastModified
)
{
CloudBlob blob = Container.GetBlobReference(oldName);
try
{
blob.FetchAttributes();
}
catch (StorageClientException e)
{
// Someone may have deleted the blob in the mean time
if (e.ErrorCode == StorageErrorCode.BlobNotFound)
{
throw new ApplicationException("Concurrency Violation", e);
}
throw;
}
BlobProperties blobProperties = blob.Properties;
//
// For directories create an empty data stream
//
if (dataStream == null)
{
dataStream = new MemoryStream();
}
SetupMetadata(blob.Metadata, fileData, relativePath);
blobProperties.ContentType = LookupMimeType(Path.GetExtension(fileData.Name));
// Specify an optimistic concurrency check to prevent races with other endpoints syncing at the same time.
BlobRequestOptions opts = new BlobRequestOptions();
opts.AccessCondition = AccessCondition.IfNotModifiedSince(expectedLastModified);
try
{
blob.UploadFromStream(dataStream, opts);
}
catch (StorageClientException e)
{
// Someone must have modified the file in the mean time
if (e.ErrorCode == StorageErrorCode.BlobNotFound || e.ErrorCode == StorageErrorCode.ConditionFailed)
{
throw new ApplicationException("Storage Error", e);
}
throw;
}
blobProperties = blob.Properties;
SyncedBlobAttributes attributes = new SyncedBlobAttributes(blob.Uri.ToString(), blobProperties.LastModifiedUtc);
return(attributes);
}
protected virtual OptimisticConcurrencyContext GetContextForUnexistentBlob(string objId)
{
return(new OptimisticConcurrencyContext()
{
ObjectId = objId,
AccessCondition = AccessCondition.IfNotModifiedSince(DateTime.MinValue)
});
}
// Called by AzureBlobSyncProvider.DeleteItem to help with removing blobs.
internal void DeleteFile(
string name,
DateTime expectedLastModified
)
{
name = name.Replace(@"\", "/");
CloudBlob blob = Container.GetBlobReference(name);
try
{
blob.FetchAttributes();
}
catch (StorageClientException e)
{
// Someone may have deleted the blob in the mean time
if (e.ErrorCode == StorageErrorCode.BlobNotFound)
{
throw new ApplicationException("Concurrency Violation", e);
}
throw;
}
BlobProperties blobProperties = blob.Properties;
bool isDirectory = bool.Parse(blob.Metadata[AzureBlobStore.IsDirectory]);
// If this is a directory then we need to look for children.
if (isDirectory)
{
IEnumerable <IListBlobItem> items = Container.ListBlobs();
if (items.Count() > 0)
{
throw new ApplicationException("Constraint Violation - Directory Not Empty");
}
}
// Specify an optimistic concurrency check to prevent races with other endpoints syncing at the same time.
BlobRequestOptions opts = new BlobRequestOptions();
opts.AccessCondition = AccessCondition.IfNotModifiedSince(expectedLastModified);
try
{
// http://social.msdn.microsoft.com/Forums/en/windowsazure/thread/08e71f7b-ca9e-4107-b5fe-06890bb512e6
//blob.Delete(opts);
blob.DeleteIfExists();
}
catch (StorageClientException e)
{
// Someone must have modified the file in the mean time
if (e.ErrorCode == StorageErrorCode.BlobNotFound || e.ErrorCode == StorageErrorCode.ConditionFailed)
{
throw new ApplicationException("Concurrency Violation", e);
}
throw;
}
}
private string UploadBlob(CloudBlob blob, Stream data, bool overwrite, string eTag)
{
BlobRequestOptions options;
if (overwrite)
{
options = String.IsNullOrEmpty(eTag) ? new BlobRequestOptions {
AccessCondition = AccessCondition.None
} : new BlobRequestOptions {
AccessCondition = AccessCondition.IfMatch(eTag)
};
}
else
{
options = new BlobRequestOptions {
AccessCondition = AccessCondition.IfNotModifiedSince(DateTime.MinValue)
};
}
try
{
if (data.CanSeek)
{
this.retryPolicy.ExecuteAction(() =>
{
data.Seek(0, SeekOrigin.Begin);
blob.UploadFromStream(data, options);
});
}
else
{
// Stream is not seekable, cannot use retry logic as data consistency cannot be guaranteed.
blob.UploadFromStream(data, options);
}
return(blob.Properties.ETag);
}
catch (StorageClientException ex)
{
if (ex.ErrorCode != StorageErrorCode.ConditionFailed)
{
throw;
}
}
return(null);
}
// Called by AzureBlobSyncProvider.InsertItem.
internal SyncedBlobAttributes InsertFile(FileData fileData, string relativePath, Stream dataStream)
{
if (fileData.Name.Length > MaxFileNameLength)
{
throw new ApplicationException("Name Too Long");
}
string path = fileData.RelativePath.ToLower();
path = path.Replace(@"\", "/");
CloudBlob blob = Container.GetBlobReference(path);
BlobProperties blobProperties = blob.Properties;
DateTime uninitTime = blobProperties.LastModifiedUtc;
SetupMetadata(blob.Metadata, fileData, relativePath);
blobProperties.ContentType = LookupMimeType(Path.GetExtension(fileData.Name));
if (fileData.IsDirectory)
{
// Directories have no stream
dataStream = new MemoryStream();
}
// Specify an optimistic concurrency check to prevent races with other endpoints syncing at the same time.
BlobRequestOptions opts = new BlobRequestOptions();
opts.AccessCondition = AccessCondition.IfNotModifiedSince(uninitTime);
try
{
blob.UploadFromStream(dataStream, opts);
}
catch (StorageException e)
{
if (e.ErrorCode == StorageErrorCode.BlobAlreadyExists || e.ErrorCode == StorageErrorCode.ConditionFailed)
{
throw new ApplicationException("Concurrency Violation", e);
}
throw;
}
blobProperties = blob.Properties;
SyncedBlobAttributes attributes = new SyncedBlobAttributes(blob.Uri.ToString(), blobProperties.LastModifiedUtc);
return(attributes);
}
// Create a file retriever for the blob with the given name.
// Note that this code attempts to handle
internal FileRetriever GetFileRetriever(
string name,
DateTime expectedLastUpdate
)
{
DateTime creationTime;
DateTime lastAccessTime;
DateTime lastWriteTime;
FileStream stream = null;
long size = 0;
string relativePath;
FileAttributes fileAttributes = 0;
string tempFileName = Path.GetTempFileName();
CloudBlob blob = Container.GetBlobReference(name);
BlobProperties props = blob.Properties;
// Specify an optimistic concurrency check to prevent races with other endpoints syncing at the same time.
BlobRequestOptions opts = new BlobRequestOptions();
opts.AccessCondition = AccessCondition.IfNotModifiedSince(expectedLastUpdate);
try
{
blob.DownloadToFile(tempFileName);
}
catch (StorageException e)
{
if (e.ErrorCode == StorageErrorCode.BlobAlreadyExists || e.ErrorCode == StorageErrorCode.ConditionFailed)
{
throw new ApplicationException("Concurrency Violation", e);
}
throw;
}
string fileName;
string pathWithName;
fileName = blob.Metadata[AzureBlobStore.FileNameKey];
creationTime = DateTime.Parse(blob.Metadata[AzureBlobStore.CreationTimeKey]);
lastAccessTime = DateTime.Parse(blob.Metadata[AzureBlobStore.LastAccessTimeKey]);
lastWriteTime = DateTime.Parse(blob.Metadata[AzureBlobStore.LastWriteTimeKey]);
relativePath = blob.Metadata[AzureBlobStore.RelativePathKey];
fileAttributes = (FileAttributes)long.Parse(blob.Metadata[AzureBlobStore.AttributesKey]);
if (!bool.Parse(blob.Metadata[AzureBlobStore.IsDirectory]))
{
// Special handling for directories
stream = File.OpenRead(tempFileName);
size = long.Parse(blob.Metadata[AzureBlobStore.SizeKey]);
}
pathWithName = blob.Metadata[AzureBlobStore.PathWithNameKey];
// Note that the IsDirectory property for the FileData object is pulled from the Attributes
FileData fd = new FileData(
pathWithName,
fileAttributes,
creationTime,
lastAccessTime,
lastWriteTime,
size);
return(new FileRetriever(fd, relativePath, stream));
}
/// <summary>
/// groom out blobs that are too old
/// </summary>
/// <param name="containerName"> </param>
/// <param name="old"> </param>
/// <param name="ct"> </param>
/// <param name="removeEmptyContainer"> </param>
public static void GroomOldBlobsFrom(string containerName, TimeSpan old, CancellationToken?ct = null,
bool removeEmptyContainer = false)
{
ILog log = ClassLogger.Create(typeof(AzureStorageAssistant));
DebugOnlyLogger dblog = DebugOnlyLogger.Create(log);
try
{
log.InfoFormat("Grooming blobs from {0} older than {1}", containerName, old.ToString());
var account =
CloudStorageAccount.FromConfigurationSetting(CommonConfiguration.DefaultStorageConnection.ToString());
account.Ensure(containers: new[] { containerName });
var bc = account.CreateCloudBlobClient();
var container = bc.GetContainerReference(containerName);
BlobRequestOptions blobQuery = new BlobRequestOptions();
blobQuery.BlobListingDetails = BlobListingDetails.None;
blobQuery.UseFlatBlobListing = true;
blobQuery.AccessCondition = AccessCondition.IfNotModifiedSince(DateTime.UtcNow.AddDays(-1 * old.Days));
if (ct.HasValue)
{
ct.Value.ThrowIfCancellationRequested();
}
IEnumerable <IListBlobItem> blobs;
blobs = container.ListBlobs(blobQuery).ToArray();
foreach (IListBlobItem blob in blobs)
{
if (ct.HasValue)
{
ct.Value.ThrowIfCancellationRequested();
}
CloudBlob cloudBlob;
cloudBlob = container.GetBlobReference(blob.Uri.ToString());
dblog.InfoFormat("Grooming blob {0}", cloudBlob.Uri);
cloudBlob.DeleteIfExists(blobQuery);
}
if (removeEmptyContainer)
{
if (!container.ListBlobs().Any())
{
container.Delete();
}
}
}
catch (Exception ex)
{
log.Warn(ex.Message);
log.Warn(ex.ToString());
throw;
}
}
