// Called by AzureBlobSyncProvider.UpdateItem to help with writing item updates internal SyncedBlobAttributes UpdateFile( string oldName, FileData fileData, string relativePath, Stream dataStream, DateTime expectedLastModified ) { CloudBlob blob = Container.GetBlobReference(oldName); try { blob.FetchAttributes(); } catch (StorageClientException e) { // Someone may have deleted the blob in the mean time if (e.ErrorCode == StorageErrorCode.BlobNotFound) { throw new ApplicationException("Concurrency Violation", e); } throw; } BlobProperties blobProperties = blob.Properties; // // For directories create an empty data stream // if (dataStream == null) { dataStream = new MemoryStream(); } SetupMetadata(blob.Metadata, fileData, relativePath); blobProperties.ContentType = LookupMimeType(Path.GetExtension(fileData.Name)); // Specify an optimistic concurrency check to prevent races with other endpoints syncing at the same time. BlobRequestOptions opts = new BlobRequestOptions(); opts.AccessCondition = AccessCondition.IfNotModifiedSince(expectedLastModified); try { blob.UploadFromStream(dataStream, opts); } catch (StorageClientException e) { // Someone must have modified the file in the mean time if (e.ErrorCode == StorageErrorCode.BlobNotFound || e.ErrorCode == StorageErrorCode.ConditionFailed) { throw new ApplicationException("Storage Error", e); } throw; } blobProperties = blob.Properties; SyncedBlobAttributes attributes = new SyncedBlobAttributes(blob.Uri.ToString(), blobProperties.LastModifiedUtc); return(attributes); }
protected virtual OptimisticConcurrencyContext GetContextForUnexistentBlob(string objId) { return(new OptimisticConcurrencyContext() { ObjectId = objId, AccessCondition = AccessCondition.IfNotModifiedSince(DateTime.MinValue) }); }
// Called by AzureBlobSyncProvider.DeleteItem to help with removing blobs. internal void DeleteFile( string name, DateTime expectedLastModified ) { name = name.Replace(@"\", "/"); CloudBlob blob = Container.GetBlobReference(name); try { blob.FetchAttributes(); } catch (StorageClientException e) { // Someone may have deleted the blob in the mean time if (e.ErrorCode == StorageErrorCode.BlobNotFound) { throw new ApplicationException("Concurrency Violation", e); } throw; } BlobProperties blobProperties = blob.Properties; bool isDirectory = bool.Parse(blob.Metadata[AzureBlobStore.IsDirectory]); // If this is a directory then we need to look for children. if (isDirectory) { IEnumerable <IListBlobItem> items = Container.ListBlobs(); if (items.Count() > 0) { throw new ApplicationException("Constraint Violation - Directory Not Empty"); } } // Specify an optimistic concurrency check to prevent races with other endpoints syncing at the same time. BlobRequestOptions opts = new BlobRequestOptions(); opts.AccessCondition = AccessCondition.IfNotModifiedSince(expectedLastModified); try { // http://social.msdn.microsoft.com/Forums/en/windowsazure/thread/08e71f7b-ca9e-4107-b5fe-06890bb512e6 //blob.Delete(opts); blob.DeleteIfExists(); } catch (StorageClientException e) { // Someone must have modified the file in the mean time if (e.ErrorCode == StorageErrorCode.BlobNotFound || e.ErrorCode == StorageErrorCode.ConditionFailed) { throw new ApplicationException("Concurrency Violation", e); } throw; } }
private string UploadBlob(CloudBlob blob, Stream data, bool overwrite, string eTag) { BlobRequestOptions options; if (overwrite) { options = String.IsNullOrEmpty(eTag) ? new BlobRequestOptions { AccessCondition = AccessCondition.None } : new BlobRequestOptions { AccessCondition = AccessCondition.IfMatch(eTag) }; } else { options = new BlobRequestOptions { AccessCondition = AccessCondition.IfNotModifiedSince(DateTime.MinValue) }; } try { if (data.CanSeek) { this.retryPolicy.ExecuteAction(() => { data.Seek(0, SeekOrigin.Begin); blob.UploadFromStream(data, options); }); } else { // Stream is not seekable, cannot use retry logic as data consistency cannot be guaranteed. blob.UploadFromStream(data, options); } return(blob.Properties.ETag); } catch (StorageClientException ex) { if (ex.ErrorCode != StorageErrorCode.ConditionFailed) { throw; } } return(null); }
// Called by AzureBlobSyncProvider.InsertItem. internal SyncedBlobAttributes InsertFile(FileData fileData, string relativePath, Stream dataStream) { if (fileData.Name.Length > MaxFileNameLength) { throw new ApplicationException("Name Too Long"); } string path = fileData.RelativePath.ToLower(); path = path.Replace(@"\", "/"); CloudBlob blob = Container.GetBlobReference(path); BlobProperties blobProperties = blob.Properties; DateTime uninitTime = blobProperties.LastModifiedUtc; SetupMetadata(blob.Metadata, fileData, relativePath); blobProperties.ContentType = LookupMimeType(Path.GetExtension(fileData.Name)); if (fileData.IsDirectory) { // Directories have no stream dataStream = new MemoryStream(); } // Specify an optimistic concurrency check to prevent races with other endpoints syncing at the same time. BlobRequestOptions opts = new BlobRequestOptions(); opts.AccessCondition = AccessCondition.IfNotModifiedSince(uninitTime); try { blob.UploadFromStream(dataStream, opts); } catch (StorageException e) { if (e.ErrorCode == StorageErrorCode.BlobAlreadyExists || e.ErrorCode == StorageErrorCode.ConditionFailed) { throw new ApplicationException("Concurrency Violation", e); } throw; } blobProperties = blob.Properties; SyncedBlobAttributes attributes = new SyncedBlobAttributes(blob.Uri.ToString(), blobProperties.LastModifiedUtc); return(attributes); }
// Create a file retriever for the blob with the given name. // Note that this code attempts to handle internal FileRetriever GetFileRetriever( string name, DateTime expectedLastUpdate ) { DateTime creationTime; DateTime lastAccessTime; DateTime lastWriteTime; FileStream stream = null; long size = 0; string relativePath; FileAttributes fileAttributes = 0; string tempFileName = Path.GetTempFileName(); CloudBlob blob = Container.GetBlobReference(name); BlobProperties props = blob.Properties; // Specify an optimistic concurrency check to prevent races with other endpoints syncing at the same time. BlobRequestOptions opts = new BlobRequestOptions(); opts.AccessCondition = AccessCondition.IfNotModifiedSince(expectedLastUpdate); try { blob.DownloadToFile(tempFileName); } catch (StorageException e) { if (e.ErrorCode == StorageErrorCode.BlobAlreadyExists || e.ErrorCode == StorageErrorCode.ConditionFailed) { throw new ApplicationException("Concurrency Violation", e); } throw; } string fileName; string pathWithName; fileName = blob.Metadata[AzureBlobStore.FileNameKey]; creationTime = DateTime.Parse(blob.Metadata[AzureBlobStore.CreationTimeKey]); lastAccessTime = DateTime.Parse(blob.Metadata[AzureBlobStore.LastAccessTimeKey]); lastWriteTime = DateTime.Parse(blob.Metadata[AzureBlobStore.LastWriteTimeKey]); relativePath = blob.Metadata[AzureBlobStore.RelativePathKey]; fileAttributes = (FileAttributes)long.Parse(blob.Metadata[AzureBlobStore.AttributesKey]); if (!bool.Parse(blob.Metadata[AzureBlobStore.IsDirectory])) { // Special handling for directories stream = File.OpenRead(tempFileName); size = long.Parse(blob.Metadata[AzureBlobStore.SizeKey]); } pathWithName = blob.Metadata[AzureBlobStore.PathWithNameKey]; // Note that the IsDirectory property for the FileData object is pulled from the Attributes FileData fd = new FileData( pathWithName, fileAttributes, creationTime, lastAccessTime, lastWriteTime, size); return(new FileRetriever(fd, relativePath, stream)); }
/// <summary> /// groom out blobs that are too old /// </summary> /// <param name="containerName"> </param> /// <param name="old"> </param> /// <param name="ct"> </param> /// <param name="removeEmptyContainer"> </param> public static void GroomOldBlobsFrom(string containerName, TimeSpan old, CancellationToken?ct = null, bool removeEmptyContainer = false) { ILog log = ClassLogger.Create(typeof(AzureStorageAssistant)); DebugOnlyLogger dblog = DebugOnlyLogger.Create(log); try { log.InfoFormat("Grooming blobs from {0} older than {1}", containerName, old.ToString()); var account = CloudStorageAccount.FromConfigurationSetting(CommonConfiguration.DefaultStorageConnection.ToString()); account.Ensure(containers: new[] { containerName }); var bc = account.CreateCloudBlobClient(); var container = bc.GetContainerReference(containerName); BlobRequestOptions blobQuery = new BlobRequestOptions(); blobQuery.BlobListingDetails = BlobListingDetails.None; blobQuery.UseFlatBlobListing = true; blobQuery.AccessCondition = AccessCondition.IfNotModifiedSince(DateTime.UtcNow.AddDays(-1 * old.Days)); if (ct.HasValue) { ct.Value.ThrowIfCancellationRequested(); } IEnumerable <IListBlobItem> blobs; blobs = container.ListBlobs(blobQuery).ToArray(); foreach (IListBlobItem blob in blobs) { if (ct.HasValue) { ct.Value.ThrowIfCancellationRequested(); } CloudBlob cloudBlob; cloudBlob = container.GetBlobReference(blob.Uri.ToString()); dblog.InfoFormat("Grooming blob {0}", cloudBlob.Uri); cloudBlob.DeleteIfExists(blobQuery); } if (removeEmptyContainer) { if (!container.ListBlobs().Any()) { container.Delete(); } } } catch (Exception ex) { log.Warn(ex.Message); log.Warn(ex.ToString()); throw; } }