public AzureStorageIndexInput(AzureStorageDirectory directory, string name)
: base($"AzureStorageIndexInput(name={Path.Combine(directory.IndexFolder, name)})")
{
_pageBlobClient = directory.BlobContainerClient.GetPageBlobClient(name);
_pageBlobClient.CreateIfNotExists(0);
_blobLength = GetBlobLength();
}
/// <summary>
/// A function which will upload pages to a specified page blob
/// </summary>
public async Task UpdatePageBlob(PageBlobClient pageBlobClient, int clientId)
{
// Get random 4 MB of data
Random random = new Random();
Byte[] sourceBytes = new Byte[1024 * 1024 * 4];
random.NextBytes(sourceBytes);
// Upload the data to the page blob at a random offset
long offset = random.Next(1024 * 1024) * 512;
Console.WriteLine("Uploading data to page blob, client: " + clientId.ToString());
// Upload 40 MB of data
int maxRequests = 10;
while (maxRequests-- > 0)
{
try
{
await pageBlobClient.UploadPagesAsync(new MemoryStream(sourceBytes), offset);
}
catch (Exception e)
{
Console.WriteLine("Client {0}, failed to upload the data {1}", clientId, e.Message);
throw e;
}
}
}
//-------------------------------------------------
// Read pages from a page blob
//-------------------------------------------------
private static void ReadFromPageBlob(PageBlobClient pageBlobClient,
long bufferOffset, long rangeSize)
{
// <Snippet_ReadFromPageBlob>
var pageBlob = pageBlobClient.Download(new HttpRange(bufferOffset, rangeSize));
// </Snippet_ReadFromPageBlob>
}
public async Task CreatePageBlob_ImmutableStorageWithVersioning()
{
// Arrange
await using DisposingImmutableStorageWithVersioningContainer vlwContainer = await GetTestVersionLevelWormContainer(TestConfigOAuth);
PageBlobClient pageBlob = InstrumentClient(vlwContainer.Container.GetPageBlobClient(GetNewBlobName()));
BlobImmutabilityPolicy immutabilityPolicy = new BlobImmutabilityPolicy
{
ExpiresOn = Recording.UtcNow.AddMinutes(5),
PolicyMode = BlobImmutabilityPolicyMode.Unlocked
};
// The service rounds Immutability Policy Expiry to the nearest second.
DateTimeOffset expectedImmutabilityPolicyExpiry = RoundToNearestSecond(immutabilityPolicy.ExpiresOn.Value);
PageBlobCreateOptions options = new PageBlobCreateOptions
{
ImmutabilityPolicy = immutabilityPolicy,
LegalHold = true
};
// Act
Response <BlobContentInfo> createResponse = await pageBlob.CreateAsync(size : Constants.KB, options);
// Assert
Response <BlobProperties> propertiesResponse = await pageBlob.GetPropertiesAsync();
Assert.AreEqual(expectedImmutabilityPolicyExpiry, propertiesResponse.Value.ImmutabilityPolicy.ExpiresOn);
Assert.AreEqual(immutabilityPolicy.PolicyMode, propertiesResponse.Value.ImmutabilityPolicy.PolicyMode);
Assert.IsTrue(propertiesResponse.Value.HasLegalHold);
}
private async Task <Uri> GetUserDelegationSasBlob(string blobName)
{
Uri serviceUri = new Uri($"https://{_accountName}.blob.core.windows.net");
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(_accountName, _accountKey);
BlobServiceClient blobServiceClient = new BlobServiceClient(serviceUri, sharedKeyCredential);
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(_containerName);
PageBlobClient pageBlobClient = containerClient.GetPageBlobClient(blobName);
// Create a SAS token that's valid for 7 days.
BlobSasBuilder blobSasBuilder = new BlobSasBuilder()
{
BlobContainerName = _containerName,
BlobName = blobName,
Resource = "b",
ExpiresOn = DateTimeOffset.UtcNow.AddDays(7)
};
blobSasBuilder.SetPermissions(BlobSasPermissions.Read);
BlobUriBuilder sasUriBuilder = new BlobUriBuilder(containerClient.Uri)
{
Query = blobSasBuilder.ToSasQueryParameters(sharedKeyCredential).ToString()
};
Uri containerSasUri = sasUriBuilder.ToUri();
BlobUriBuilder blobUriBuilder = new BlobUriBuilder(containerSasUri)
{
BlobName = pageBlobClient.Name
};
return(blobUriBuilder.ToUri());
}
public async Task <PageBlobClient> CreatePageBlobClientAsync(BlobContainerClient container, long size)
{
PageBlobClient blob = InstrumentClient(container.GetPageBlobClient(GetNewBlobName()));
await blob.CreateAsync(size, 0).ConfigureAwait(false);
return(blob);
}
/// <summary>
/// This method copies the changes since the last snapshot to the target base blob
/// </summary>
/// <param name="backupStorageAccountBlobClient">An instance of BlobClient which represents the storage account where the base blob is stored.</param>
/// <param name="targetContainerName">The name of container in the target storage account where the base blob is stored</param>
/// <param name="targetBaseBlobName">The name of the base blob used for storing the backups in the target storage account </param>
/// <param name="lastSnapshotSASUri">The SAS URI of the last incremental snapshot</param>
/// <param name="currentSnapshotSASUri">The SAS URI of the current snapshot</param>
/// <returns></returns>
private static async Task CopyChangesSinceLastSnapshotToBaseBlob(PageBlobClient targetBaseBlob, string lastSnapshotSASUri, string currentSnapshotSASUri)
{
PageBlobClient snapshot = new PageBlobClient(new Uri(currentSnapshotSASUri));
//Get the changes since the last incremental snapshots of the managed disk.
//GetManagedDiskDiffAsync is a new method introduced to get the changes since the last snapshot
PageRangesInfo pageRangesInfo = await snapshot.GetManagedDiskPageRangesDiffAsync(null, null, new Uri(lastSnapshotSASUri), null, CancellationToken.None);
foreach (var range in pageRangesInfo.ClearRanges)
{
await targetBaseBlob.ClearPagesAsync(range);
}
foreach (var range in pageRangesInfo.PageRanges)
{
Int64 rangeSize = (Int64)(range.Length);
// Chop a range into 4MB chunchs
for (Int64 subOffset = 0; subOffset < rangeSize; subOffset += FourMegabyteAsBytes)
{
int subRangeSize = (int)Math.Min(rangeSize - subOffset, FourMegabyteAsBytes);
HttpRange sourceRange = new HttpRange(subOffset, subRangeSize);
//When you use WritePagesAsync by passing the SAS URI of the source snapshot, the SDK uses Put Page From URL rest API: https://docs.microsoft.com/en-us/rest/api/storageservices/put-page-from-url
//When this API is invoked, the Storage service reads the data from source and copies the data to the target blob without requiring clients to buffer the data.
await targetBaseBlob.UploadPagesFromUriAsync(new Uri(currentSnapshotSASUri), sourceRange, sourceRange, null, null, null, CancellationToken.None);
}
}
await targetBaseBlob.CreateSnapshotAsync();
}
//-------------------------------------------------
// Resize page blob
//-------------------------------------------------
private static void ResizePageBlob(PageBlobClient pageBlobClient,
long OneGigabyteAsBytes)
{
// <Snippet_ResizePageBlob>
pageBlobClient.Resize(32 * OneGigabyteAsBytes);
// </Snippet_ResizePageBlob>
}
public static PageBlobClient WithCustomerProvidedKey(
this PageBlobClient blob,
CustomerProvidedKey customerProvidedKey) =>
new PageBlobClient(
ToHttps(blob.Uri),
blob.Pipeline,
blob.ClientDiagnostics,
customerProvidedKey);
//-------------------------------------------------
// Write pages to a page blob
//-------------------------------------------------
private static void WriteToPageBlob(PageBlobClient pageBlobClient,
Stream dataStream)
{
long startingOffset = 512;
// <Snippet_WriteToPageBlob>
pageBlobClient.UploadPages(dataStream, startingOffset);
// </Snippet_WriteToPageBlob>
}
protected void SendString(PageBlobClient blob, string message)
{
byte[] msgBytes = encoding.GetBytes(message);
using (MemoryStream ms = new MemoryStream(msgBytes))
{
blob.UploadPages(ms, 0);
}
}
protected void SendStream(PageBlobClient blob, Stream stream)
{
if (!overwrite && blob.Exists())
{
return;
}
blob.UploadPages(stream, 0);
}
//TODO remove ToHttps() after service fixes HTTPS bug.
public static PageBlobClient WithEncryptionScope(
this PageBlobClient blob,
string encryptionScope) =>
new PageBlobClient(
ToHttps(blob.Uri),
blob.Pipeline,
blob.Version,
blob.ClientDiagnostics,
null,
encryptionScope);
// Convert Track1 Blob object to Track 2 page blob Client
public static PageBlobClient GetTrack2PageBlobClient(CloudBlob cloubBlob, AzureStorageContext context, BlobClientOptions options = null)
{
PageBlobClient blobClient = (PageBlobClient)Util.GetTrack2BlobClientWithType(
AzureStorageBlob.GetTrack2BlobClient(cloubBlob, context, options),
context,
Track2blobModel.BlobType.Page,
options);
return(blobClient);
}
private async Task <byte[]> DownloadRange(PageBlobClient client, HttpRange range)
{
var memoryStream = new MemoryStream();
using BlobDownloadStreamingResult result1 = await client.DownloadStreamingAsync(range : range);
await result1.Content.CopyToAsync(memoryStream);
return(memoryStream.ToArray());
}
static async Task Main(string[] args)
{
//The subscription Id where the incremental snapshots of the managed disk are created
string subscriptionId = "yourSubscriptionId";
//The resource group name where incremental snapshots of the managed disks are created
string resourceGroupName = "yourResourceGroupName";
//The name of the disk that is backed up with incremental snapshots in the source region
string diskName = "yourManagedDiskName";
//The name of the storage account in the target region where incremental snapshots from source region are copied to a base blob.
string targetStorageAccountName = "yourTargetStorageAccountName";
string targetStorageConnectionString = "targetStorageConnectionString";
//the name of the container where base blob is stored on the target storage account
string targetContainerName = "yourcontainername";
//the name of the base VHD (blob) used for storing the backups in the target storage account
string targetBaseBlobName = "yourtargetbaseblobname.vhd";
//Get the incremental snapshots associated
//The incremental snapshots are already sorted in the ascending order of the created datetime
List <Snapshot> incrementalSnapshots = await GetIncrementalSnapshots(subscriptionId, resourceGroupName, diskName);
//Get the SAS URI of the first snapshot that will be used to copy it to the back Storage account
string firstSnapshotSASURI = GetSASURI(subscriptionId, resourceGroupName, incrementalSnapshots[0].Name);
//Instantiate the page blob client
PageBlobClient targetBaseBlobClient = await InstantiateBasePageBlobClient(targetStorageConnectionString, targetContainerName, targetBaseBlobName);
//Copy the first snapshot as base blob in the target storage account
await CopyFirstSnapshotToBackupStorageAccount(targetBaseBlobClient, firstSnapshotSASURI);
//The first incremental snapshot is the previous snapshot for the second incremental snapshot
string previousSnapshotSASUri = firstSnapshotSASURI;
//Remove the first snapshot as it is already copied to the base blob.
incrementalSnapshots.Remove(incrementalSnapshots[0]);
//Loop through each incremental snapshots from second to the last
foreach (var isnapshot in incrementalSnapshots)
{
//Get the SAS URI of the current snapshot
string currentSnapshotSASURI = GetSASURI(subscriptionId, resourceGroupName, isnapshot.Name);
//Copy the changes since the last snapshot to the target base blob
await CopyChangesSinceLastSnapshotToBaseBlob(targetBaseBlobClient, previousSnapshotSASUri, currentSnapshotSASURI);
//Set the current snapshot as the previous snapshot for the next snapshot
previousSnapshotSASUri = currentSnapshotSASURI;
}
}
public async Task PageBlobSample()
{
// Instantiate a new BlobServiceClient using a connection string.
BlobServiceClient blobServiceClient = new BlobServiceClient(TestConfigurations.DefaultTargetTenant.ConnectionString);
// Instantiate a new BlobContainerClient
BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient($"mycontainer3-{Guid.NewGuid()}");
try
{
// Create new Container in the Service
await blobContainerClient.CreateAsync();
// Instantiate a new PageBlobClient
PageBlobClient pageBlobClient = blobContainerClient.GetPageBlobClient("pageblob");
// Create PageBlob in the Service
const int blobSize = 1024;
await pageBlobClient.CreateAsync(size : blobSize);
// Upload content to PageBlob
using (FileStream fileStream = File.OpenRead("Samples/SampleSource.txt"))
{
// Because the file size varies slightly across platforms
// and PageBlob pages need to be multiples of 512, we'll
// pad the file to our blobSize
using (MemoryStream pageStream = new MemoryStream(new byte[blobSize]))
{
await fileStream.CopyToAsync(pageStream);
pageStream.Seek(0, SeekOrigin.Begin);
await pageBlobClient.UploadPagesAsync(
content : pageStream,
offset : 0);
}
}
// Download PageBlob
using (FileStream fileStream = File.Create("PageDestination.txt"))
{
Response <BlobDownloadInfo> downloadResponse = await pageBlobClient.DownloadAsync();
await downloadResponse.Value.Content.CopyToAsync(fileStream);
}
// Delete PageBlob in the Service
await pageBlobClient.DeleteAsync();
}
finally
{
// Delete Container in the Service
await blobContainerClient.DeleteAsync();
}
}
public static PageBlobClient WithCustomerProvidedKey(
this PageBlobClient blob,
CustomerProvidedKey customerProvidedKey) =>
new PageBlobClient(
ToHttps(blob.Uri),
blob.Pipeline,
blob.SharedKeyCredential,
blob.Version,
blob.ClientDiagnostics,
customerProvidedKey,
null);
protected void SendBytes(PageBlobClient blob, byte[] msgBytes)
{
if (!overwrite && blob.Exists())
{
return;
}
using (MemoryStream ms = new MemoryStream(msgBytes))
{
blob.UploadPages(ms, 0);
}
}
/// <summary>
/// Basic operations to work with page blobs
/// </summary>
/// <returns>A Task object.</returns>
private static async Task BasicStoragePageBlobOperationsAsync()
{
const string PageBlobName = "samplepageblob";
string containerName = ContainerPrefix + Guid.NewGuid();
// Retrieve storage account information from connection string
BlobServiceClient blobServiceClient = Common.CreateblobServiceClientFromConnectionString();
// Create a container for organizing blobs within the storage account.
Console.WriteLine("1. Creating Container");
BlobContainerClient container = blobServiceClient.GetBlobContainerClient(containerName);
await container.CreateIfNotExistsAsync();
// Create a page blob in the newly created container.
Console.WriteLine("2. Creating Page Blob");
PageBlobClient pageBlob = container.GetPageBlobClient(PageBlobName);
await pageBlob.CreateAsync(512 * 2 /*size*/); // size needs to be multiple of 512 bytes
// Write to a page blob
Console.WriteLine("3. Write to a Page Blob");
byte[] samplePagedata = new byte[512];
Random random = new Random();
random.NextBytes(samplePagedata);
using (var stream = new MemoryStream(samplePagedata))
{
await pageBlob.UploadPagesAsync(stream, 0);
}
// List all blobs in this container. Because a container can contain a large number of blobs the results
// are returned in segments with a maximum of 5000 blobs per segment. You can define a smaller maximum segment size
// using the maxResults parameter on ListBlobsSegmentedAsync.
Console.WriteLine("4. List Blobs in Container");
var resultSegment = container.GetBlobsAsync();
await foreach (var blob in resultSegment)
{
// Blob type will be BlobClient, CloudPageBlob or BlobClientDirectory
Console.WriteLine("{0} (type: {1}", blob.Name, blob.GetType());
}
// Read from a page blob
Console.WriteLine("5. Read from a Page Blob");
var httpRange = new HttpRange(0, samplePagedata.Count());
var downloadInfo = await pageBlob.DownloadAsync(httpRange);
// Clean up after the demo
Console.WriteLine("6. Delete page Blob");
await pageBlob.DeleteIfExistsAsync();
Console.WriteLine("7. Delete Container");
await container.DeleteIfExistsAsync();
}
//-------------------------------------------------
// Read valid page regions from a page blob
//-------------------------------------------------
private static void ReadValidPageRegionsFromPageBlob(PageBlobClient pageBlobClient,
long bufferOffset, long rangeSize)
{
// <Snippet_ReadValidPageRegionsFromPageBlob>
IEnumerable <HttpRange> pageRanges = pageBlobClient.GetPageRanges().Value.PageRanges;
foreach (var range in pageRanges)
{
var pageBlob = pageBlobClient.Download(range);
}
// </Snippet_ReadValidPageRegionsFromPageBlob>
}
/// <summary>
/// Instantiate an instance of BlobClient which is used to perform common operations such as creating containers, blobs e.t.c. in a storage account
/// </summary>
/// <param name="storageAccountName">The name of a storage account</param>
/// <param name="storageAccountSASToken">The SAS token of a storage account</param>
/// <returns></returns>
private static async Task <PageBlobClient> InstantiateBasePageBlobClient(string connectionString, string targetContainerName, string targetBaseBlobName)
{
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
//Create the target container if not already exist
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(targetContainerName);
await containerClient.CreateIfNotExistsAsync();
PageBlobClient targetBaseBlob = containerClient.GetPageBlobClient(targetBaseBlobName);
return(targetBaseBlob);
}
public async Task Capture_Span_When_Create_Page_Blob()
{
await using var scope = await BlobContainerScope.CreateContainer(Environment.StorageAccountConnectionString);
var blobName = Guid.NewGuid().ToString();
var client = new PageBlobClient(Environment.StorageAccountConnectionString, scope.ContainerName, blobName);
await Agent.Tracer.CaptureTransaction("Create Azure Page Blob", ApiConstants.TypeStorage, async() =>
{
var blobCreateResponse = await client.CreateAsync(1024);
});
AssertSpan("Create", $"{scope.ContainerName}/{blobName}");
}
public PageBlobWriteStream(
PageBlobClient pageBlobClient,
long bufferSize,
long position,
PageBlobRequestConditions conditions,
IProgress <long> progressHandler) : base(
position,
bufferSize,
progressHandler)
{
ValidateBufferSize(bufferSize);
ValidatePosition(position);
_pageBlobClient = pageBlobClient;
_conditions = conditions ?? new PageBlobRequestConditions();
_writeIndex = position;
}
/// <summary>
/// This method writes the page ranges from the source snapshot in the source region to the target base blob in the target region
/// </summary>
/// <param name="sourceSnapshotSASUri">The SAS URI of the source snapshot</param>
/// <param name="targetBaseBlob">An instance of CloudPageBlob which represents the target base blob</param>
/// <param name="pageRanges">Page ranges on the source snapshots that have changed since the last snapshot</param>
/// <returns></returns>
private static async Task WritePageRanges(string sourceSnapshotSASUri, PageBlobClient targetBaseBlob, PageRangesInfo pageRangeInfo)
{
foreach (HttpRange range in pageRangeInfo.PageRanges)
{
Int64 rangeSize = (Int64)(range.Length);
// Chop a range into 4MB chunchs
for (Int64 subOffset = 0; subOffset < rangeSize; subOffset += FourMegabyteAsBytes)
{
int subRangeSize = (int)Math.Min(rangeSize - subOffset, FourMegabyteAsBytes);
HttpRange sourceRange = new HttpRange(subOffset, subRangeSize);
await targetBaseBlob.UploadPagesFromUriAsync(new Uri(sourceSnapshotSASUri), sourceRange, sourceRange, null, null, null, CancellationToken.None);
}
}
}
protected void SendFile(PageBlobClient blob, string fileNameAndPath)
{
if (!File.Exists(fileNameAndPath))
{
throw new CantSendFileDataWhenFileDoesNotExistException(fileNameAndPath);
}
if (!overwrite && blob.Exists())
{
return;
}
using (StreamReader sr = new StreamReader(fileNameAndPath))
{
blob.UploadPages(sr.BaseStream, 0);
}
}
public async Task PageBlobSample()
{
// Instantiate a new BlobServiceClient using a connection string.
BlobServiceClient blobServiceClient = new BlobServiceClient(TestConfigurations.DefaultTargetTenant.ConnectionString);
// Instantiate a new BlobContainerClient
BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient("mycontainer3");
try
{
// Create new Container in the Service
await blobContainerClient.CreateAsync();
// Instantiate a new PageBlobClient
PageBlobClient pageBlobClient = blobContainerClient.GetPageBlobClient("pageblob");
// Create PageBlob in the Service
await pageBlobClient.CreateAsync(size : 1024);
// Upload content to PageBlob
using (FileStream fileStream = File.OpenRead("Samples/SampleSource.txt"))
{
await pageBlobClient.UploadPagesAsync(
content : fileStream,
offset : 0);
}
// Download PageBlob
using (FileStream fileStream = File.Create("PageDestination.txt"))
{
Response <BlobDownloadInfo> downloadResponse = await pageBlobClient.DownloadAsync();
await downloadResponse.Value.Content.CopyToAsync(fileStream);
}
// Delete PageBlob in the Service
await pageBlobClient.DeleteAsync();
}
finally
{
// Delete Container in the Service
await blobContainerClient.DeleteAsync();
}
}
public GetPageRangesAsyncCollection(
bool diff,
PageBlobClient client,
HttpRange?range,
string snapshot,
string previousSnapshot,
Uri previousSnapshotUri,
PageBlobRequestConditions requestConditions,
string operationName)
{
_diff = diff;
_client = client;
_range = range;
_snapshot = snapshot;
_previousSnapshot = previousSnapshot;
_previousSnapshotUri = previousSnapshotUri;
_requestConditions = requestConditions;
_operationName = operationName;
}
public async Task Capture_Span_When_Upload_Page_Blob()
{
await using var scope = await BlobContainerScope.CreateContainer(Environment.StorageAccountConnectionString);
var blobName = Guid.NewGuid().ToString();
var client = new PageBlobClient(Environment.StorageAccountConnectionString, scope.ContainerName, blobName);
var blobCreateResponse = await client.CreateAsync(1024);
await Agent.Tracer.CaptureTransaction("Upload Azure Page Blob", ApiConstants.TypeStorage, async() =>
{
var random = new Random();
var bytes = new byte[512];
random.NextBytes(bytes);
var stream = new MemoryStream(bytes);
var uploadPagesResponse = await client.UploadPagesAsync(stream, 0);
});
AssertSpan("Upload", $"{scope.ContainerName}/{blobName}");
}
protected override async Task <BlobBaseClient> GetBlobClient(string blobName, bool createIfNotExists = false)
{
PageBlobClient blob = null;
if (this.Uri != null)
{
var pageUri = this.Uri.Combine(this.FolderName, blobName);
blob = new PageBlobClient(pageUri, this.Credential);
}
else
{
blob = new PageBlobClient(this.ConnectionString, this.ContainerName, Path.Combine(this.FolderName, blobName));
}
if (createIfNotExists)
{
await blob.CreateIfNotExistsAsync(1024).ConfigureAwait(false);
}
return(blob);
}
