/// <summary> /// Upload File to blob with storage Client library API /// </summary> internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagement localChannel, string filePath, StorageBlob.CloudBlob blob) { BlobClientOptions options = null; if (this.Force.IsPresent || !blob.Exists() || ShouldContinue(string.Format(Resources.OverwriteConfirmation, blob.Uri), null)) { // Prepare blob Properties, MetaData, accessTier BlobHttpHeaders blobHttpHeaders = CreateBlobHttpHeaders(BlobProperties); IDictionary <string, string> metadata = new Dictionary <string, string>(); SetBlobMeta_Track2(metadata, this.Metadata); AccessTier?accesstier = GetAccessTier_Track2(this.standardBlobTier, this.pageBlobTier); //Prepare progress handler long fileSize = new FileInfo(ResolvedFileName).Length; string activity = String.Format(Resources.SendAzureBlobActivity, this.File, blob.Name, blob.Container.Name); string status = Resources.PrepareUploadingBlob; ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status); IProgress <long> progressHandler = new Progress <long>((finishedBytes) => { if (pr != null) { // Size of the source file might be 0, when it is, directly treat the progress as 100 percent. pr.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize); pr.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, pr.PercentComplete); Console.WriteLine(finishedBytes); this.OutputStream.WriteProgress(pr); } }); using (FileStream stream = System.IO.File.OpenRead(ResolvedFileName)) { //block blob if (string.Equals(blobType, BlockBlobType, StringComparison.InvariantCultureIgnoreCase)) { BlobClient blobClient = GetTrack2BlobClient(blob, localChannel.StorageContext, options); StorageTransferOptions trasnferOption = new StorageTransferOptions() { MaximumConcurrency = this.GetCmdletConcurrency() }; BlobUploadOptions uploadOptions = new BlobUploadOptions(); uploadOptions.Metadata = metadata; uploadOptions.HttpHeaders = blobHttpHeaders; uploadOptions.Conditions = this.BlobRequestConditions; uploadOptions.AccessTier = accesstier; uploadOptions.ProgressHandler = progressHandler; uploadOptions.TransferOptions = trasnferOption; await blobClient.UploadAsync(stream, uploadOptions, CmdletCancellationToken).ConfigureAwait(false); } //Page or append blob else if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) || string.Equals(blobType, AppendBlobType, StringComparison.InvariantCultureIgnoreCase)) { PageBlobClient pageblobClient = null; AppendBlobClient appendblobClient = null; //Create Blob if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page { if (fileSize % 512 != 0) { throw new ArgumentException(String.Format("File size {0} Bytes is invalid for PageBlob, must be a multiple of 512 bytes.", fileSize.ToString())); } pageblobClient = GetTrack2PageBlobClient(blob, localChannel.StorageContext, options); PageBlobCreateOptions createOptions = new PageBlobCreateOptions(); createOptions.Metadata = metadata; createOptions.HttpHeaders = blobHttpHeaders; createOptions.Conditions = this.PageBlobRequestConditions; Response <BlobContentInfo> blobInfo = await pageblobClient.CreateAsync(fileSize, createOptions, CmdletCancellationToken).ConfigureAwait(false); } else //append { appendblobClient = GetTrack2AppendBlobClient(blob, localChannel.StorageContext, options); AppendBlobCreateOptions createOptions = new AppendBlobCreateOptions(); createOptions.Metadata = metadata; createOptions.HttpHeaders = blobHttpHeaders; createOptions.Conditions = this.AppendBlobRequestConditions; Response <BlobContentInfo> blobInfo = await appendblobClient.CreateAsync(createOptions, CmdletCancellationToken).ConfigureAwait(false); } // Upload blob content byte[] uploadcache4MB = null; byte[] uploadcache = null; progressHandler.Report(0); long offset = 0; while (offset < fileSize) { // Get chunk size and prepare cache int chunksize = size4MB; if (chunksize <= (fileSize - offset)) // Chunk size will be 4MB { if (uploadcache4MB == null) { uploadcache4MB = new byte[size4MB]; } uploadcache = uploadcache4MB; } else // last chunk can < 4MB { chunksize = (int)(fileSize - offset); if (uploadcache4MB == null) { uploadcache = new byte[chunksize]; } else { uploadcache = uploadcache4MB; } } //Get content to upload for the chunk int readoutcount = await stream.ReadAsync(uploadcache, 0, (int)chunksize).ConfigureAwait(false); MemoryStream chunkContent = new MemoryStream(uploadcache, 0, readoutcount); //Upload content if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page { Response <PageInfo> pageInfo = await pageblobClient.UploadPagesAsync(chunkContent, offset, null, null, null, CmdletCancellationToken).ConfigureAwait(false); } else //append { Response <BlobAppendInfo> pageInfo = await appendblobClient.AppendBlockAsync(chunkContent, null, null, null, CmdletCancellationToken).ConfigureAwait(false); } // Update progress offset += readoutcount; progressHandler.Report(offset); } if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) && accesstier != null) { await pageblobClient.SetAccessTierAsync(accesstier.Value, cancellationToken : CmdletCancellationToken).ConfigureAwait(false); } } else { throw new InvalidOperationException(string.Format( CultureInfo.CurrentCulture, Resources.InvalidBlobType, blobType, BlobName)); } } WriteCloudBlobObject(taskId, localChannel, blob); } }
public async Task CloudBlobSoftDeleteSnapshotTask() { CloudBlobContainer container = GetRandomContainerReference(); try { //Enables a delete retention policy on the blob with 1 day of default retention days container.ServiceClient.EnableSoftDelete(); await container.CreateAsync(); // Upload some data to the blob. var blobName = GetRandomBlobName(); MemoryStream originalData = new MemoryStream(GetRandomBuffer(1024)); CloudAppendBlob appendBlob = container.GetAppendBlobReference(blobName); await appendBlob.UploadFromStreamAsync(originalData); CloudBlob blob = container.GetBlobReference(blobName); //create snapshot via api CloudBlob snapshot = blob.Snapshot(); //create snapshot via write protection await appendBlob.UploadFromStreamAsync(originalData); //we should have 2 snapshots 1 regular and 1 deleted: there is no way to get only the deleted snapshots but the below listing will get both snapshot types int blobCount = 0; int deletedSnapshotCount = 0; int snapShotCount = 0; BlobContinuationToken ct = null; do { Task <BlobResultSegment> resultSegments = container.ListBlobsSegmentedAsync (null, true, BlobListingDetails.Snapshots | BlobListingDetails.Deleted, null, ct, null, null); List <IListBlobItem> blobs = resultSegments.Result .Results .ToList(); ct = resultSegments.Result.ContinuationToken; foreach (IListBlobItem item in blobs) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, blobName); if (blobItem.IsSnapshot) { snapShotCount++; } if (blobItem.IsDeleted) { Assert.IsNotNull(blobItem.Properties.DeletedTime); Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0); deletedSnapshotCount++; } blobCount++; } } while (ct != null); Assert.AreEqual(3, blobCount); Assert.AreEqual(deletedSnapshotCount, 1); Assert.AreEqual(snapShotCount, 2); blobCount = 0; ct = null; do { foreach (IListBlobItem item in (await container.ListBlobsSegmentedAsync (null, true, BlobListingDetails.Snapshots, null, ct, null, null)) .Results .ToList()) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, blobName); Assert.IsFalse(blobItem.IsDeleted); Assert.IsNull(blobItem.Properties.DeletedTime); Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete); blobCount++; } } while (ct != null); Assert.AreEqual(blobCount, 2); //Delete Blob and snapshots blob.Delete(DeleteSnapshotsOption.IncludeSnapshots); Assert.IsFalse(blob.Exists()); Assert.IsFalse(snapshot.Exists()); blobCount = 0; ct = null; do { foreach (IListBlobItem item in (await container.ListBlobsSegmentedAsync (null, true, BlobListingDetails.All, null, ct, null, null)) .Results .ToList()) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, blobName); Assert.IsTrue(blobItem.IsDeleted); Assert.IsNotNull(blobItem.Properties.DeletedTime); Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0); blobCount++; } }while(ct != null); Assert.AreEqual(blobCount, 3); blob.Undelete(); blob.FetchAttributes(); Assert.IsFalse(blob.IsDeleted); Assert.IsNull(blob.Properties.DeletedTime); Assert.IsNull(blob.Properties.RemainingDaysBeforePermanentDelete); blobCount = 0; ct = null; do { foreach (IListBlobItem item in (await container.ListBlobsSegmentedAsync (null, true, BlobListingDetails.All, null, ct, null, null)) .Results .ToList()) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, blobName); Assert.IsFalse(blobItem.IsDeleted); Assert.IsNull(blobItem.Properties.DeletedTime); Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete); blobCount++; } }while(ct != null); Assert.AreEqual(blobCount, 3); } finally { container.ServiceClient.DisableSoftDelete(); await container.DeleteAsync(); } }
public void CloudBlobSoftDeleteNoSnapshot() { CloudBlobContainer container = GetRandomContainerReference(); try { //Enables a delete retention policy on the blob with 1 day of default retention days container.ServiceClient.EnableSoftDelete(); Shared.Protocol.ServiceProperties props = container.ServiceClient.GetServiceProperties(); container.Create(); // Upload some data to the blob. var blobName = GetRandomBlobName(); MemoryStream originalData = new MemoryStream(GetRandomBuffer(1024)); CloudAppendBlob appendBlob = container.GetAppendBlobReference(blobName); appendBlob.CreateOrReplace(); appendBlob.AppendBlock(originalData, null); CloudBlob blob = container.GetBlobReference(blobName); Assert.IsTrue(blob.Exists()); Assert.IsFalse(blob.IsDeleted); blob.Delete(); Assert.IsFalse(blob.Exists()); int blobCount = 0; foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.Snapshots | BlobListingDetails.Deleted)) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, blobName); Assert.IsTrue(blobItem.IsDeleted); Assert.IsNotNull(blobItem.Properties.DeletedTime); Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0); blobCount++; } Assert.AreEqual(blobCount, 1); blob.Undelete(); blob.FetchAttributes(); Assert.IsFalse(blob.IsDeleted); Assert.IsNull(blob.Properties.DeletedTime); Assert.IsNull(blob.Properties.RemainingDaysBeforePermanentDelete); blobCount = 0; foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All)) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, blobName); Assert.IsFalse(blobItem.IsDeleted); Assert.IsNull(blobItem.Properties.DeletedTime); Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete); blobCount++; } Assert.AreEqual(blobCount, 1); } finally { container.ServiceClient.DisableSoftDelete(); container.DeleteIfExists(); } }
public void CloudBlobSoftDeleteNoSnapshotAPM() { CloudBlobContainer container = GetRandomContainerReference(); try { //Enables a delete retention policy on the blob with 1 day of default retention days container.ServiceClient.EnableSoftDelete(); container.Create(); // Upload some data to the blob. var blobName = GetRandomBlobName(); MemoryStream originalData = new MemoryStream(GetRandomBuffer(1024)); CloudAppendBlob appendBlob = container.GetAppendBlobReference(blobName); appendBlob.CreateOrReplace(); appendBlob.AppendBlock(originalData, null); CloudBlob blob = container.GetBlobReference(blobName); Assert.IsFalse(blob.IsDeleted); IAsyncResult result; using (AutoResetEvent waitHandle = new AutoResetEvent(false)) { result = blob.BeginDelete(DeleteSnapshotsOption.None, null, null, null, ar => waitHandle.Set(), null); waitHandle.WaitOne(); blob.EndDelete(result); } Assert.IsFalse(blob.Exists()); int blobCount = 0; foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All)) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, blobName); Assert.IsTrue(blobItem.IsDeleted); Assert.IsNotNull(blobItem.Properties.DeletedTime); Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0); blobCount++; } Assert.AreEqual(1, blobCount); using (AutoResetEvent waitHandle = new AutoResetEvent(false)) { result = blob.BeginUndelete(ar => waitHandle.Set(), null); waitHandle.WaitOne(); blob.EndUndelete(result); } blob.FetchAttributes(); Assert.IsFalse(blob.IsDeleted); Assert.IsNull(blob.Properties.DeletedTime); Assert.IsNull(blob.Properties.RemainingDaysBeforePermanentDelete); blobCount = 0; foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All)) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, blobName); Assert.IsFalse(blobItem.IsDeleted); Assert.IsNull(blobItem.Properties.DeletedTime); Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete); blobCount++; } Assert.AreEqual(blobCount, 1); } finally { container.ServiceClient.DisableSoftDelete(); container.DeleteIfExists(); } }
public void CloudBlobSoftDeleteSnapshot() { CloudBlobContainer container = GetRandomContainerReference(); try { //Enables a delete retention policy on the blob with 1 day of default retention days container.ServiceClient.EnableSoftDelete(); container.Create(); // Upload some data to the blob. MemoryStream originalData = new MemoryStream(GetRandomBuffer(1024)); CloudAppendBlob appendBlob = container.GetAppendBlobReference(BlobName); appendBlob.UploadFromStream(originalData); CloudBlob blob = container.GetBlobReference(BlobName); //create snapshot via api CloudBlob snapshot = blob.Snapshot(); //create snapshot via write protection appendBlob.UploadFromStream(originalData); //we should have 2 snapshots 1 regular and 1 deleted: there is no way to get only the deleted snapshots but the below listing will get both snapshot types int blobCount = 0; int deletedSnapshotCount = 0; int snapShotCount = 0; foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.Snapshots | BlobListingDetails.Deleted)) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, BlobName); if (blobItem.IsSnapshot) { snapShotCount++; } if (blobItem.IsDeleted) { Assert.IsNotNull(blobItem.Properties.DeletedTime); Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0); deletedSnapshotCount++; } blobCount++; } Assert.AreEqual(blobCount, 3); Assert.AreEqual(deletedSnapshotCount, 1); Assert.AreEqual(snapShotCount, 2); blobCount = 0; foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.Snapshots)) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, BlobName); Assert.IsFalse(blobItem.IsDeleted); Assert.IsNull(blobItem.Properties.DeletedTime); Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete); blobCount++; } Assert.AreEqual(blobCount, 2); //Delete Blob and snapshots blob.Delete(DeleteSnapshotsOption.IncludeSnapshots); Assert.IsFalse(blob.Exists()); Assert.IsFalse(snapshot.Exists()); blobCount = 0; foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All)) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, BlobName); Assert.IsTrue(blobItem.IsDeleted); Assert.IsNotNull(blobItem.Properties.DeletedTime); Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0); blobCount++; } Assert.AreEqual(blobCount, 3); blob.Undelete(); blob.FetchAttributes(); Assert.IsFalse(blob.IsDeleted); Assert.IsNull(blob.Properties.DeletedTime); Assert.IsNull(blob.Properties.RemainingDaysBeforePermanentDelete); blobCount = 0; foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All)) { CloudAppendBlob blobItem = (CloudAppendBlob)item; Assert.AreEqual(blobItem.Name, BlobName); Assert.IsFalse(blobItem.IsDeleted); Assert.IsNull(blobItem.Properties.DeletedTime); Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete); blobCount++; } Assert.AreEqual(blobCount, 3); } finally { container.ServiceClient.DisableSoftDelete(); container.DeleteIfExists(); } }