public async Task CreatePageBlob_ImmutableStorageWithVersioning()
{
// Arrange
await using DisposingImmutableStorageWithVersioningContainer vlwContainer = await GetTestVersionLevelWormContainer(TestConfigOAuth);
PageBlobClient pageBlob = InstrumentClient(vlwContainer.Container.GetPageBlobClient(GetNewBlobName()));
BlobImmutabilityPolicy immutabilityPolicy = new BlobImmutabilityPolicy
{
ExpiresOn = Recording.UtcNow.AddMinutes(5),
PolicyMode = BlobImmutabilityPolicyMode.Unlocked
};
// The service rounds Immutability Policy Expiry to the nearest second.
DateTimeOffset expectedImmutabilityPolicyExpiry = RoundToNearestSecond(immutabilityPolicy.ExpiresOn.Value);
PageBlobCreateOptions options = new PageBlobCreateOptions
{
ImmutabilityPolicy = immutabilityPolicy,
LegalHold = true
};
// Act
Response <BlobContentInfo> createResponse = await pageBlob.CreateAsync(size : Constants.KB, options);
// Assert
Response <BlobProperties> propertiesResponse = await pageBlob.GetPropertiesAsync();
Assert.AreEqual(expectedImmutabilityPolicyExpiry, propertiesResponse.Value.ImmutabilityPolicy.ExpiresOn);
Assert.AreEqual(immutabilityPolicy.PolicyMode, propertiesResponse.Value.ImmutabilityPolicy.PolicyMode);
Assert.IsTrue(propertiesResponse.Value.HasLegalHold);
}
public async Task <PageBlobClient> CreatePageBlobClientAsync(BlobContainerClient container, long size)
{
PageBlobClient blob = InstrumentClient(container.GetPageBlobClient(GetNewBlobName()));
await blob.CreateAsync(size, 0).ConfigureAwait(false);
return(blob);
}
public async Task PageBlobSample()
{
// Instantiate a new BlobServiceClient using a connection string.
BlobServiceClient blobServiceClient = new BlobServiceClient(TestConfigurations.DefaultTargetTenant.ConnectionString);
// Instantiate a new BlobContainerClient
BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient($"mycontainer3-{Guid.NewGuid()}");
try
{
// Create new Container in the Service
await blobContainerClient.CreateAsync();
// Instantiate a new PageBlobClient
PageBlobClient pageBlobClient = blobContainerClient.GetPageBlobClient("pageblob");
// Create PageBlob in the Service
const int blobSize = 1024;
await pageBlobClient.CreateAsync(size : blobSize);
// Upload content to PageBlob
using (FileStream fileStream = File.OpenRead("Samples/SampleSource.txt"))
{
// Because the file size varies slightly across platforms
// and PageBlob pages need to be multiples of 512, we'll
// pad the file to our blobSize
using (MemoryStream pageStream = new MemoryStream(new byte[blobSize]))
{
await fileStream.CopyToAsync(pageStream);
pageStream.Seek(0, SeekOrigin.Begin);
await pageBlobClient.UploadPagesAsync(
content : pageStream,
offset : 0);
}
}
// Download PageBlob
using (FileStream fileStream = File.Create("PageDestination.txt"))
{
Response <BlobDownloadInfo> downloadResponse = await pageBlobClient.DownloadAsync();
await downloadResponse.Value.Content.CopyToAsync(fileStream);
}
// Delete PageBlob in the Service
await pageBlobClient.DeleteAsync();
}
finally
{
// Delete Container in the Service
await blobContainerClient.DeleteAsync();
}
}
/// <summary>
/// Basic operations to work with page blobs
/// </summary>
/// <returns>A Task object.</returns>
private static async Task BasicStoragePageBlobOperationsAsync()
{
const string PageBlobName = "samplepageblob";
string containerName = ContainerPrefix + Guid.NewGuid();
// Retrieve storage account information from connection string
BlobServiceClient blobServiceClient = Common.CreateblobServiceClientFromConnectionString();
// Create a container for organizing blobs within the storage account.
Console.WriteLine("1. Creating Container");
BlobContainerClient container = blobServiceClient.GetBlobContainerClient(containerName);
await container.CreateIfNotExistsAsync();
// Create a page blob in the newly created container.
Console.WriteLine("2. Creating Page Blob");
PageBlobClient pageBlob = container.GetPageBlobClient(PageBlobName);
await pageBlob.CreateAsync(512 * 2 /*size*/); // size needs to be multiple of 512 bytes
// Write to a page blob
Console.WriteLine("3. Write to a Page Blob");
byte[] samplePagedata = new byte[512];
Random random = new Random();
random.NextBytes(samplePagedata);
using (var stream = new MemoryStream(samplePagedata))
{
await pageBlob.UploadPagesAsync(stream, 0);
}
// List all blobs in this container. Because a container can contain a large number of blobs the results
// are returned in segments with a maximum of 5000 blobs per segment. You can define a smaller maximum segment size
// using the maxResults parameter on ListBlobsSegmentedAsync.
Console.WriteLine("4. List Blobs in Container");
var resultSegment = container.GetBlobsAsync();
await foreach (var blob in resultSegment)
{
// Blob type will be BlobClient, CloudPageBlob or BlobClientDirectory
Console.WriteLine("{0} (type: {1}", blob.Name, blob.GetType());
}
// Read from a page blob
Console.WriteLine("5. Read from a Page Blob");
var httpRange = new HttpRange(0, samplePagedata.Count());
var downloadInfo = await pageBlob.DownloadAsync(httpRange);
// Clean up after the demo
Console.WriteLine("6. Delete page Blob");
await pageBlob.DeleteIfExistsAsync();
Console.WriteLine("7. Delete Container");
await container.DeleteIfExistsAsync();
}
public async Task Capture_Span_When_Create_Page_Blob()
{
await using var scope = await BlobContainerScope.CreateContainer(Environment.StorageAccountConnectionString);
var blobName = Guid.NewGuid().ToString();
var client = new PageBlobClient(Environment.StorageAccountConnectionString, scope.ContainerName, blobName);
await Agent.Tracer.CaptureTransaction("Create Azure Page Blob", ApiConstants.TypeStorage, async() =>
{
var blobCreateResponse = await client.CreateAsync(1024);
});
AssertSpan("Create", $"{scope.ContainerName}/{blobName}");
}
public async Task PageBlobSample()
{
// Instantiate a new BlobServiceClient using a connection string.
BlobServiceClient blobServiceClient = new BlobServiceClient(TestConfigurations.DefaultTargetTenant.ConnectionString);
// Instantiate a new BlobContainerClient
BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient("mycontainer3");
try
{
// Create new Container in the Service
await blobContainerClient.CreateAsync();
// Instantiate a new PageBlobClient
PageBlobClient pageBlobClient = blobContainerClient.GetPageBlobClient("pageblob");
// Create PageBlob in the Service
await pageBlobClient.CreateAsync(size : 1024);
// Upload content to PageBlob
using (FileStream fileStream = File.OpenRead("Samples/SampleSource.txt"))
{
await pageBlobClient.UploadPagesAsync(
content : fileStream,
offset : 0);
}
// Download PageBlob
using (FileStream fileStream = File.Create("PageDestination.txt"))
{
Response <BlobDownloadInfo> downloadResponse = await pageBlobClient.DownloadAsync();
await downloadResponse.Value.Content.CopyToAsync(fileStream);
}
// Delete PageBlob in the Service
await pageBlobClient.DeleteAsync();
}
finally
{
// Delete Container in the Service
await blobContainerClient.DeleteAsync();
}
}
public async Task Capture_Span_When_Upload_Page_Blob()
{
await using var scope = await BlobContainerScope.CreateContainer(Environment.StorageAccountConnectionString);
var blobName = Guid.NewGuid().ToString();
var client = new PageBlobClient(Environment.StorageAccountConnectionString, scope.ContainerName, blobName);
var blobCreateResponse = await client.CreateAsync(1024);
await Agent.Tracer.CaptureTransaction("Upload Azure Page Blob", ApiConstants.TypeStorage, async() =>
{
var random = new Random();
var bytes = new byte[512];
random.NextBytes(bytes);
var stream = new MemoryStream(bytes);
var uploadPagesResponse = await client.UploadPagesAsync(stream, 0);
});
AssertSpan("Upload", $"{scope.ContainerName}/{blobName}");
}
/// <summary>
/// This method copies the first incremental snapshot as a base blob in the target region
/// </summary>
/// <param name="backupStorageAccountBlobClient">An instance of BlobClient which represents the storage account where the base blob is stored.</param>
/// <param name="targetContainerName">The name of container in the target storage account where the base blob is stored</param>
/// <param name="targetBaseBlobName">The name of the base blob used for storing the backups in the target storage account </param>
/// <param name="sourceSnapshotSASUri">The SAS URI of the source snapshot</param>
/// <returns></returns>
private static async Task CopyFirstSnapshotToBackupStorageAccount(PageBlobClient targetBaseBlob, string sourceSnapshotSASUri)
{
PageBlobClient sourceSnapshot = new PageBlobClient(new Uri(sourceSnapshotSASUri));
//Get the size of the source snapshot
var snapshotProperties = await sourceSnapshot.GetPropertiesAsync();
long sourceSnapshotSize = snapshotProperties.Value.ContentLength;
//Create the target base blob with the same size as the source snapshot
await targetBaseBlob.CreateAsync(sourceSnapshotSize);
//Snapshots are stored as page blobs under the hood
//Get all the valid page ranges from the source snapshot.
//Learn more about page blobs and page ranges:
//https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-pageblob-overview
///https://blogs.msdn.microsoft.com/windowsazurestorage/2012/03/26/getting-the-page-ranges-of-a-large-page-blob-in-segments/
///https://docs.microsoft.com/en-us/rest/api/storageservices/get-page-ranges
PageRangesInfo pageRanges = await sourceSnapshot.GetPageRangesAsync();
await WritePageRanges(sourceSnapshotSASUri, targetBaseBlob, pageRanges);
await targetBaseBlob.CreateSnapshotAsync();
}
/// <summary>
/// Upload File to blob with storage Client library API
/// </summary>
internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagement localChannel, string filePath, StorageBlob.CloudBlob blob)
{
BlobClientOptions options = null;
if (this.Force.IsPresent ||
!blob.Exists() ||
ShouldContinue(string.Format(Resources.OverwriteConfirmation, blob.Uri), null))
{
// Prepare blob Properties, MetaData, accessTier
BlobHttpHeaders blobHttpHeaders = CreateBlobHttpHeaders(BlobProperties);
IDictionary <string, string> metadata = new Dictionary <string, string>();
SetBlobMeta_Track2(metadata, this.Metadata);
AccessTier?accesstier = GetAccessTier_Track2(this.standardBlobTier, this.pageBlobTier);
//Prepare progress handler
long fileSize = new FileInfo(ResolvedFileName).Length;
string activity = String.Format(Resources.SendAzureBlobActivity, this.File, blob.Name, blob.Container.Name);
string status = Resources.PrepareUploadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
IProgress <long> progressHandler = new Progress <long>((finishedBytes) =>
{
if (pr != null)
{
// Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
pr.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize);
pr.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, pr.PercentComplete);
Console.WriteLine(finishedBytes);
this.OutputStream.WriteProgress(pr);
}
});
using (FileStream stream = System.IO.File.OpenRead(ResolvedFileName))
{
//block blob
if (string.Equals(blobType, BlockBlobType, StringComparison.InvariantCultureIgnoreCase))
{
BlobClient blobClient = GetTrack2BlobClient(blob, localChannel.StorageContext, options);
StorageTransferOptions trasnferOption = new StorageTransferOptions()
{
MaximumConcurrency = this.GetCmdletConcurrency()
};
BlobUploadOptions uploadOptions = new BlobUploadOptions();
uploadOptions.Metadata = metadata;
uploadOptions.HttpHeaders = blobHttpHeaders;
uploadOptions.Conditions = this.BlobRequestConditions;
uploadOptions.AccessTier = accesstier;
uploadOptions.ProgressHandler = progressHandler;
uploadOptions.TransferOptions = trasnferOption;
await blobClient.UploadAsync(stream, uploadOptions, CmdletCancellationToken).ConfigureAwait(false);
}
//Page or append blob
else if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) ||
string.Equals(blobType, AppendBlobType, StringComparison.InvariantCultureIgnoreCase))
{
PageBlobClient pageblobClient = null;
AppendBlobClient appendblobClient = null;
//Create Blob
if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page
{
if (fileSize % 512 != 0)
{
throw new ArgumentException(String.Format("File size {0} Bytes is invalid for PageBlob, must be a multiple of 512 bytes.", fileSize.ToString()));
}
pageblobClient = GetTrack2PageBlobClient(blob, localChannel.StorageContext, options);
PageBlobCreateOptions createOptions = new PageBlobCreateOptions();
createOptions.Metadata = metadata;
createOptions.HttpHeaders = blobHttpHeaders;
createOptions.Conditions = this.PageBlobRequestConditions;
Response <BlobContentInfo> blobInfo = await pageblobClient.CreateAsync(fileSize, createOptions, CmdletCancellationToken).ConfigureAwait(false);
}
else //append
{
appendblobClient = GetTrack2AppendBlobClient(blob, localChannel.StorageContext, options);
AppendBlobCreateOptions createOptions = new AppendBlobCreateOptions();
createOptions.Metadata = metadata;
createOptions.HttpHeaders = blobHttpHeaders;
createOptions.Conditions = this.AppendBlobRequestConditions;
Response <BlobContentInfo> blobInfo = await appendblobClient.CreateAsync(createOptions, CmdletCancellationToken).ConfigureAwait(false);
}
// Upload blob content
byte[] uploadcache4MB = null;
byte[] uploadcache = null;
progressHandler.Report(0);
long offset = 0;
while (offset < fileSize)
{
// Get chunk size and prepare cache
int chunksize = size4MB;
if (chunksize <= (fileSize - offset)) // Chunk size will be 4MB
{
if (uploadcache4MB == null)
{
uploadcache4MB = new byte[size4MB];
}
uploadcache = uploadcache4MB;
}
else // last chunk can < 4MB
{
chunksize = (int)(fileSize - offset);
if (uploadcache4MB == null)
{
uploadcache = new byte[chunksize];
}
else
{
uploadcache = uploadcache4MB;
}
}
//Get content to upload for the chunk
int readoutcount = await stream.ReadAsync(uploadcache, 0, (int)chunksize).ConfigureAwait(false);
MemoryStream chunkContent = new MemoryStream(uploadcache, 0, readoutcount);
//Upload content
if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page
{
Response <PageInfo> pageInfo = await pageblobClient.UploadPagesAsync(chunkContent, offset, null, null, null, CmdletCancellationToken).ConfigureAwait(false);
}
else //append
{
Response <BlobAppendInfo> pageInfo = await appendblobClient.AppendBlockAsync(chunkContent, null, null, null, CmdletCancellationToken).ConfigureAwait(false);
}
// Update progress
offset += readoutcount;
progressHandler.Report(offset);
}
if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) && accesstier != null)
{
await pageblobClient.SetAccessTierAsync(accesstier.Value, cancellationToken : CmdletCancellationToken).ConfigureAwait(false);
}
}
else
{
throw new InvalidOperationException(string.Format(
CultureInfo.CurrentCulture,
Resources.InvalidBlobType,
blobType,
BlobName));
}
}
WriteCloudBlobObject(taskId, localChannel, blob);
}
}
