public BlobDecryptStream(Stream userStream, IDictionary<string, string> metadata, long? userProvidedLength, int discardFirst, bool bufferIV, bool noPadding, BlobEncryptionPolicy policy, bool? requireEncryption)
{
this.userStream = userStream;
this.metadata = metadata;
this.userProvidedLength = userProvidedLength;
this.discardFirst = discardFirst;
this.encryptionPolicy = policy;
this.bufferIV = bufferIV;
this.noPadding = noPadding;
this.requireEncryption = requireEncryption;
}
static void Main(string[] args)
{
// This is standard code to interact with Blob Storage
StorageCredentials creds = new StorageCredentials(
ConfigurationManager.AppSettings["accountName"],
ConfigurationManager.AppSettings["accountKey"]);
CloudStorageAccount account = new CloudStorageAccount(creds, useHttps: true);
CloudBlobClient client = account.CreateCloudBlobClient();
CloudBlobContainer contain = client.GetContainerReference(ConfigurationManager.AppSettings["container"]);
contain.CreateIfNotExists();
// The Resolver object is used to interact with Key Vault for Azure Storage
// This is where the GetToken method from above is used
KeyVaultKeyResolver cloudResolver = new KeyVaultKeyResolver(GetToken);
// Retrieve the key that you created previously
// The IKey that is returned here is an RsaKey
// Remember that we used the names contosokeyvault and testrsakey1
var rsa = cloudResolver.ResolveKeyAsync("https://contosokeyvault.vault.azure.net/keys/TestRSAKey1", CancellationToken.None).GetAwaiter().GetResult();
// Now you simply use the RSA key to encrypt by setting it in the BlobEncryptionPolicy.
BlobEncryptionPolicy policy = new BlobEncryptionPolicy(rsa, null);
BlobRequestOptions options = new BlobRequestOptions() { EncryptionPolicy = policy };
// Reference a block blob
CloudBlockBlob blob = contain.GetBlockBlobReference("MyFile.txt");
// Upload using the UploadFromStream method
using (var stream = System.IO.File.OpenRead(@"C:\data\MyFile.txt"))
blob.UploadFromStream(stream, stream.Length, null, options, null);
// In this case we will not pass a key and only pass the resolver because
// this policy will only be used for downloading / decrypting
policy = new BlobEncryptionPolicy(null, cloudResolver);
options = new BlobRequestOptions() { EncryptionPolicy = policy };
using (var np = File.Open(@"C:\data\MyFileDecrypted.txt", FileMode.Create))
blob.DownloadToStream(np, null, options, null);
}
//protected CancellationToken cancellationToken;
public LargeDownloadToFileSettings(CloudBlob blob, string filePath, FileMode fileMode, long?offset, long?length, int parallelIOCount = 16,
long maxRangeSizeInBytes = 100 *1024 *1024, TimeSpan?maxExecutionTimePerRange = null, IRetryPolicy retryPolicy = null, BlobEncryptionPolicy encryptionPolicy = null, bool?requireEncryption = null,
LocationMode?locationMode = null)
{
//CommonUtility.AssertNotNull("blob", blob);
//CommonUtility.AssertNotNull("filePath", filePath);
//CommonUtility.AssertInBounds("parallelIOCount", parallelIOCount, 1, int.MaxValue);
this.Blob = blob;
this.FilePath = filePath;
this.FileMode = fileMode;
this.Offset = offset;
this.Length = length;
this.ParallelIOCount = parallelIOCount;
this.MaxRangeSizeInBytes = maxRangeSizeInBytes;
this.MaxExecutionTimePerRange = maxExecutionTimePerRange;
this.RetryPolicy = retryPolicy;
this.EncryptionPolicy = encryptionPolicy;
this.RequireEncryption = requireEncryption;
this.LocationMode = locationMode;
}
private void DoCloudBlobEncryption(BlobType type, bool partial)
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer = GetRandomBuffer(size);
if (partial)
{
size = 2 * 1024 * 1024;
}
ICloudBlob blob;
if (type == BlobType.BlockBlob)
{
blob = container.GetBlockBlobReference("blockblob");
}
else if (type == BlobType.PageBlob)
{
blob = container.GetPageBlobReference("pageblob");
}
else
{
blob = container.GetAppendBlobReference("appendblob");
}
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = new DictionaryKeyResolver();
resolver.Add(aesKey);
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
MemoryStream stream;
// Upload the encrypted contents to the blob.
using (stream = new MemoryStream(buffer))
{
blob.UploadFromStream(stream, size, null, uploadOptions, null);
// Ensure that the user stream is open.
Assert.IsTrue(stream.CanSeek);
}
// Download the encrypted blob.
// Create the decryption policy to be used for download. There is no need to specify the
// key when the policy is only going to be used for downloads. Resolver is sufficient.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
// Download and decrypt the encrypted contents from the blob.
MemoryStream outputStream = new MemoryStream();
blob.DownloadToStream(outputStream, null, downloadOptions, null);
// Ensure that the user stream is open.
outputStream.Seek(0, SeekOrigin.Begin);
// Compare that the decrypted contents match the input data.
byte[] outputArray = outputStream.ToArray();
TestHelper.AssertBuffersAreEqualUptoIndex(outputArray, buffer, size - 1);
}
finally
{
container.DeleteIfExists();
}
}
private void DoBlobEncryptedWriteStreamTestAPM(BlobType type)
{
byte[] buffer = GetRandomBuffer(8 * 1024);
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
ICloudBlob blob = null;
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
OperationContext opContext = new OperationContext();
CloudBlobStream blobStream = null;
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
IAsyncResult result;
if (type == BlobType.BlockBlob)
{
blob = container.GetBlockBlobReference("blob1");
blob.StreamWriteSizeInBytes = 16 * 1024;
result = ((CloudBlockBlob)blob).BeginOpenWrite(null, uploadOptions, opContext, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blobStream = ((CloudBlockBlob)blob).EndOpenWrite(result);
}
else if (type == BlobType.PageBlob)
{
blob = container.GetPageBlobReference("blob1");
blob.StreamWriteSizeInBytes = 16 * 1024;
result = ((CloudPageBlob)blob).BeginOpenWrite(40 * 1024, null, uploadOptions, opContext, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blobStream = ((CloudPageBlob)blob).EndOpenWrite(result);
}
else
{
blob = container.GetAppendBlobReference("blob1");
blob.StreamWriteSizeInBytes = 16 * 1024;
result = ((CloudAppendBlob)blob).BeginOpenWrite(true, null, uploadOptions, opContext, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blobStream = ((CloudAppendBlob)blob).EndOpenWrite(result);
}
}
using (MemoryStream wholeBlob = new MemoryStream())
{
using (blobStream)
{
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
IAsyncResult result;
for (int i = 0; i < 3; i++)
{
result = blobStream.BeginWrite(
buffer,
0,
buffer.Length,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
blobStream.EndWrite(result);
wholeBlob.Write(buffer, 0, buffer.Length);
}
// Append and Page blobs have one extra call due to create.
if (type == BlobType.BlockBlob)
{
Assert.AreEqual(1, opContext.RequestResults.Count);
}
else
{
Assert.AreEqual(2, opContext.RequestResults.Count);
}
result = blobStream.BeginWrite(
buffer,
0,
buffer.Length,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
blobStream.EndWrite(result);
wholeBlob.Write(buffer, 0, buffer.Length);
result = blobStream.BeginWrite(
buffer,
0,
buffer.Length,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
blobStream.EndWrite(result);
wholeBlob.Write(buffer, 0, buffer.Length);
// Append and Page blobs have one extra call due to create.
if (type == BlobType.BlockBlob)
{
Assert.AreEqual(2, opContext.RequestResults.Count);
}
else
{
Assert.AreEqual(3, opContext.RequestResults.Count);
}
result = blobStream.BeginCommit(
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
blobStream.EndCommit(result);
Assert.AreEqual(4, opContext.RequestResults.Count);
}
}
Assert.AreEqual(4, opContext.RequestResults.Count);
using (MemoryStream downloadedBlob = new MemoryStream())
{
blob.DownloadToStream(downloadedBlob, null, uploadOptions);
TestHelper.AssertStreamsAreEqual(wholeBlob, downloadedBlob);
}
}
}
finally
{
container.DeleteIfExists();
}
}
public void CloudBlockBlobValidateEncryptionAPM()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer = GetRandomBuffer(size);
CloudBlockBlob blob = container.GetBlockBlobReference("blob1");
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
// Upload the encrypted contents to the blob.
MemoryStream stream = new MemoryStream(buffer);
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
ICancellableAsyncResult result = blob.BeginUploadFromStream(
stream, size, null, uploadOptions, null, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blob.EndUploadFromStream(result);
}
// Encrypt locally.
CryptoStream encryptedStream;
using (AesCryptoServiceProvider myAes = new AesCryptoServiceProvider())
{
string metadata = blob.Metadata[Constants.EncryptionConstants.BlobEncryptionData];
BlobEncryptionData encryptionData = JsonConvert.DeserializeObject<BlobEncryptionData>(metadata);
myAes.IV = encryptionData.ContentEncryptionIV;
myAes.Key = aesKey.UnwrapKeyAsync(encryptionData.WrappedContentKey.EncryptedKey, encryptionData.WrappedContentKey.Algorithm, CancellationToken.None).Result;
stream.Seek(0, SeekOrigin.Begin);
encryptedStream = new CryptoStream(stream, myAes.CreateEncryptor(), CryptoStreamMode.Read);
}
// Download the encrypted blob.
MemoryStream outputStream = new MemoryStream();
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
ICancellableAsyncResult result = blob.BeginDownloadToStream(outputStream, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blob.EndDownloadToStream(result);
}
outputStream.Seek(0, SeekOrigin.Begin);
for (int i = 0; i < outputStream.Length; i++)
{
Assert.AreEqual(encryptedStream.ReadByte(), outputStream.ReadByte());
}
}
finally
{
container.DeleteIfExists();
}
}
private void ValidateRangeDecryption(BlobType type, int blobSize, int? blobOffset, int? length, int? verifyLength = null)
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
byte[] buffer = GetRandomBuffer(blobSize);
ICloudBlob blob = GetCloudBlobReference(type, container);
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions options = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
// Upload the encrypted contents to the blob.
MemoryStream stream = new MemoryStream(buffer);
blob.UploadFromStream(stream, blobSize, null, options, null);
// Download a range in the encrypted blob.
MemoryStream outputStream = new MemoryStream();
blob.DownloadRangeToStream(outputStream, blobOffset, length, null, options, null);
outputStream.Seek(0, SeekOrigin.Begin);
if (length.HasValue)
{
if (verifyLength.HasValue)
{
Assert.AreEqual(verifyLength.Value, outputStream.Length);
}
else
{
Assert.AreEqual(length.Value, outputStream.Length);
}
}
// Compare that the decrypted contents match the input data.
byte[] outputArray = outputStream.ToArray();
for (int i = 0; i < outputArray.Length; i++)
{
int bufferOffset = (int) (blobOffset.HasValue ? blobOffset : 0);
Assert.AreEqual(buffer[bufferOffset + i], outputArray[i]);
}
}
finally
{
container.DeleteIfExists();
}
}
public void CloudBlobEncryptionWithText()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
string data = "String data";
CloudBlockBlob blob = container.GetBlockBlobReference("blockblob");
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = new DictionaryKeyResolver();
resolver.Add(aesKey);
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
// Upload the encrypted contents to the blob.
blob.UploadText(data, null, null, uploadOptions, null);
// Download the encrypted blob.
// Create the decryption policy to be used for download. There is no need to specify the
// key when the policy is only going to be used for downloads. Resolver is sufficient.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
// Download and decrypt the encrypted contents from the blob.
string outputData = blob.DownloadText(null, null, downloadOptions, null);
// Compare that the decrypted contents match the input data.
Assert.AreEqual(data, outputData);
}
finally
{
container.DeleteIfExists();
}
}
private static void DoCloudBlockBlobEncryptionValidateWrappers(IKey key, DictionaryKeyResolver keyResolver)
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer = GetRandomBuffer(size);
CloudBlockBlob blob = container.GetBlockBlobReference("blob1");
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(key, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
MemoryStream stream;
// Upload the encrypted contents to the blob.
using (stream = new MemoryStream(buffer))
{
blob.UploadFromStream(stream, size, null, uploadOptions, null);
// Ensure that the user stream is open.
Assert.IsTrue(stream.CanSeek);
}
// Download the encrypted blob.
// Create the decryption policy to be used for download. There is no need to specify the encryption mode
// and the key wrapper when the policy is only going to be used for downloads.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, keyResolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
// Download and decrypt the encrypted contents from the blob.
MemoryStream outputStream = new MemoryStream();
blob.DownloadToStream(outputStream, null, downloadOptions, null);
// Ensure that the user stream is open.
outputStream.Seek(0, SeekOrigin.Begin);
// Compare that the decrypted contents match the input data.
byte[] outputArray = outputStream.ToArray();
TestHelper.AssertBuffersAreEqual(outputArray, buffer);
}
finally
{
container.DeleteIfExists();
}
}
static void Main(string[] args)
{
Console.WriteLine("Blob encryption with Key Vault integration");
Console.WriteLine();
// Get the key ID from App.config if it exists.
string keyID = CloudConfigurationManager.GetSetting("KeyID");
// If no key ID was specified, we will create a new secret in Key Vault.
// To create a new secret, this client needs full permission to Key Vault secrets.
// Once the secret is created, its ID can be added to App.config. Once this is done,
// this client only needs read access to secrets.
if (string.IsNullOrEmpty(keyID))
{
Console.WriteLine("No secret specified in App.config.");
Console.WriteLine("Please enter the name of a new secret to create in Key Vault.");
Console.WriteLine("WARNING: This will delete any existing secret with the same name.");
Console.Write("Name of the new secret to create [{0}]: ", DefaultSecretName);
string newSecretName = Console.ReadLine().Trim();
if (string.IsNullOrEmpty(newSecretName))
{
newSecretName = DefaultSecretName;
}
// Although it is possible to use keys (rather than secrets) stored in Key Vault, this prevents caching.
// Therefore it is recommended to use secrets along with a caching resolver (see below).
keyID = EncryptionShared.KeyVaultUtility.SetUpKeyVaultSecret(newSecretName);
Console.WriteLine();
Console.WriteLine("Created a secret with ID: {0}", keyID);
Console.WriteLine("Copy the secret ID to App.config to reuse.");
Console.WriteLine();
}
// Retrieve storage account information from connection string
// How to create a storage connection string - https://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/
CloudStorageAccount storageAccount = EncryptionShared.Utility.CreateStorageAccountFromConnectionString();
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(DemoContainer + Guid.NewGuid().ToString("N"));
// Construct a resolver capable of looking up keys and secrets stored in Key Vault.
KeyVaultKeyResolver cloudResolver = new KeyVaultKeyResolver(EncryptionShared.KeyVaultUtility.GetAccessToken);
// To demonstrate how multiple different types of key can be used, we also create a local key and resolver.
// This key is temporary and won't be persisted.
RsaKey rsaKey = new RsaKey("rsakey");
LocalResolver resolver = new LocalResolver();
resolver.Add(rsaKey);
// If there are multiple key sources like Azure Key Vault and local KMS, set up an aggregate resolver as follows.
// This helps users to define a plug-in model for all the different key providers they support.
AggregateKeyResolver aggregateResolver = new AggregateKeyResolver()
.Add(resolver)
.Add(cloudResolver);
// Set up a caching resolver so the secrets can be cached on the client. This is the recommended usage
// pattern since the throttling targets for Storage and Key Vault services are orders of magnitude
// different.
CachingKeyResolver cachingResolver = new CachingKeyResolver(2, aggregateResolver);
// Create a key instance corresponding to the key ID. This will cache the secret.
IKey cloudKey = cachingResolver.ResolveKeyAsync(keyID, CancellationToken.None).GetAwaiter().GetResult();
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer = new byte[size];
Random rand = new Random();
rand.NextBytes(buffer);
// The first blob will use the key stored in Azure Key Vault.
CloudBlockBlob blob = container.GetBlockBlobReference("blockblob1");
// Create the encryption policy using the secret stored in Azure Key Vault to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(cloudKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
Console.WriteLine("Uploading the 1st encrypted blob.");
// Upload the encrypted contents to the blob.
using (MemoryStream stream = new MemoryStream(buffer))
{
blob.UploadFromStream(stream, size, null, uploadOptions, null);
}
// Download the encrypted blob.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, cachingResolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
Console.WriteLine("Downloading the 1st encrypted blob.");
// Download and decrypt the encrypted contents from the blob.
using (MemoryStream outputStream = new MemoryStream())
{
blob.DownloadToStream(outputStream, null, downloadOptions, null);
}
// Upload second blob using the local key.
blob = container.GetBlockBlobReference("blockblob2");
// Create the encryption policy using the local key.
uploadPolicy = new BlobEncryptionPolicy(rsaKey, null);
// Set the encryption policy on the request options.
uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
Console.WriteLine("Uploading the 2nd encrypted blob.");
// Upload the encrypted contents to the blob.
using (MemoryStream stream = new MemoryStream(buffer))
{
blob.UploadFromStream(stream, size, null, uploadOptions, null);
}
// Download the encrypted blob. The same policy and options created before can be used because the aggregate resolver contains both
// resolvers and will pick the right one based on the key ID stored in blob metadata on the service.
Console.WriteLine("Downloading the 2nd encrypted blob.");
// Download and decrypt the encrypted contents from the blob.
using (MemoryStream outputStream = new MemoryStream())
{
blob.DownloadToStream(outputStream, null, downloadOptions, null);
}
Console.WriteLine("Press enter key to exit");
Console.ReadLine();
}
finally
{
container.DeleteIfExists();
}
}
public void CloudBlobEncryptionOpenWriteStreamAPMWithAccessCondition()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
int size = (int)(1 * Constants.MB);
container.Create();
byte[] buffer = GetRandomBuffer(size);
CloudBlockBlob blob = container.GetBlockBlobReference("blockblob");
blob.StreamWriteSizeInBytes = (int)(4 * Constants.MB);
blob.UploadText("Sample initial text");
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = new DictionaryKeyResolver();
resolver.Add(aesKey);
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions()
{
EncryptionPolicy = uploadPolicy,
RequireEncryption = true
};
AccessCondition accessCondition = AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag);
using (MemoryStream stream = new NonSeekableMemoryStream(buffer))
{
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
blob.EndUploadFromStream(blob.BeginUploadFromStream(
stream, size, accessCondition, uploadOptions, null, ar => waitHandle.Set(), null));
}
}
// Download the encrypted blob.
// Create the decryption policy to be used for download. There is no need to specify the
// key when the policy is only going to be used for downloads. Resolver is sufficient.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions()
{
EncryptionPolicy = downloadPolicy,
RequireEncryption = true
};
// Download and decrypt the encrypted contents from the blob.
MemoryStream outputStream = new MemoryStream();
blob.DownloadToStream(outputStream, null, downloadOptions, null);
// Ensure that the user stream is open.
outputStream.Seek(0, SeekOrigin.Begin);
// Compare that the decrypted contents match the input data.
byte[] outputArray = outputStream.ToArray();
TestHelper.AssertBuffersAreEqualUptoIndex(outputArray, buffer, size - 1);
}
finally
{
container.DeleteIfExists();
}
}
private void DoCloudBlobEncryptionWithStrictMode(BlobType type)
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer = GetRandomBuffer(size);
ICloudBlob blob;
if (type == BlobType.BlockBlob)
{
blob = container.GetBlockBlobReference("blob1");
}
else
{
blob = container.GetPageBlobReference("blob1");
}
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = new DictionaryKeyResolver();
resolver.Add(aesKey);
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
// Set RequireEncryption flag to true.
uploadOptions.RequireEncryption = true;
// Upload an encrypted blob with the policy set.
MemoryStream stream = new MemoryStream(buffer);
blob.UploadFromStream(stream, size, null, uploadOptions, null);
// Upload the blob when RequireEncryption is true and no policy is set. This should throw an error.
uploadOptions.EncryptionPolicy = null;
stream = new MemoryStream(buffer);
TestHelper.ExpectedException<InvalidOperationException>(
() => blob.UploadFromStream(stream, size, null, uploadOptions, null),
"Not specifying a policy when RequireEnryption is set to true should throw.");
// Create the encryption policy to be used for download.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
// Set the encryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
// Set RequireEncryption flag to true.
downloadOptions.RequireEncryption = true;
// Download the encrypted blob.
MemoryStream outputStream = new MemoryStream();
blob.DownloadToStream(outputStream, null, downloadOptions, null);
blob.Metadata.Clear();
// Upload a plain text blob.
stream = new MemoryStream(buffer);
blob.UploadFromStream(stream, size);
// Try to download an encrypted blob with RequireEncryption set to true. This should throw.
outputStream = new MemoryStream();
TestHelper.ExpectedException<StorageException>(
() => blob.DownloadToStream(outputStream, null, downloadOptions, null),
"Downloading with RequireEncryption set to true and no metadata on the service should fail.");
// Set RequireEncryption to false and download.
downloadOptions.RequireEncryption = false;
blob.DownloadToStream(outputStream, null, downloadOptions, null);
}
finally
{
container.DeleteIfExists();
}
}
private static void DoCloudBlobEncryptionAPM(BlobType type, bool partial)
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer = GetRandomBuffer(size);
if (partial)
{
size = 2 * 1024 * 1024;
}
ICloudBlob blob = GetCloudBlobReference(type, container);
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = new DictionaryKeyResolver();
resolver.Add(aesKey);
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
MemoryStream stream;
// Upload the encrypted contents to the blob.
using (stream = new MemoryStream(buffer))
{
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
ICancellableAsyncResult result = blob.BeginUploadFromStream(
stream, size, null, uploadOptions, null, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blob.EndUploadFromStream(result);
}
// Ensure that the user stream is open.
Assert.IsTrue(stream.CanSeek);
}
// Download the encrypted blob.
// Create the decryption policy to be used for download. There is no need to specify the encryption mode
// and the key wrapper when the policy is only going to be used for downloads.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
// Download and decrypt the encrypted contents from the blob.
MemoryStream outputStream = new MemoryStream();
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
ICancellableAsyncResult result = blob.BeginDownloadToStream(outputStream, null, downloadOptions, null, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blob.EndDownloadToStream(result);
}
// Ensure that the user stream is open.
outputStream.Seek(0, SeekOrigin.Begin);
// Compare that the decrypted contents match the input data.
byte[] outputArray = outputStream.ToArray();
TestHelper.AssertBuffersAreEqualUptoIndex(outputArray, buffer, size - 1);
}
finally
{
container.DeleteIfExists();
}
}
public void BlobUploadWorksWithDefaultRequestOptions()
{
CloudBlobContainer container = GetRandomContainerReference();
byte[] buffer = GetRandomBuffer(16 * 1024);
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy policy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions options = new BlobRequestOptions() { EncryptionPolicy = policy };
// Set default request options
container.ServiceClient.DefaultRequestOptions = options;
try
{
container.Create();
using (MemoryStream stream = new MemoryStream(buffer))
{
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blockblob");
blockBlob.UploadFromStream(stream, buffer.Length);
}
}
finally
{
container.DeleteIfExists();
}
}
public void BlobUpdateShouldThrowWithEncryption()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
byte[] buffer = GetRandomBuffer(16 * 1024);
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
using (MemoryStream stream = new MemoryStream(buffer))
{
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blockblob");
TestHelper.ExpectedException<InvalidOperationException>(
() => blockBlob.PutBlock(Convert.ToBase64String(Guid.NewGuid().ToByteArray()), stream, null, null, uploadOptions, null),
"PutBlock does not support encryption.");
CloudAppendBlob appendBlob = container.GetAppendBlobReference("appendblob");
TestHelper.ExpectedException<InvalidOperationException>(
() => appendBlob.AppendBlock(stream, null, null, uploadOptions, null),
"AppendBlock does not support encryption.");
CloudPageBlob pageBlob = container.GetPageBlobReference("pageblob");
TestHelper.ExpectedException<InvalidOperationException>(
() => pageBlob.WritePages(stream, 0, null, null, uploadOptions, null),
"WritePages does not support encryption.");
TestHelper.ExpectedException<InvalidOperationException>(
() => pageBlob.ClearPages(0, 512, null, uploadOptions, null),
"ClearPages does not support encryption.");
}
}
finally
{
container.DeleteIfExists();
}
}
private void CountOperationsHelper(int size, int targetUploadOperations, bool streamSeekable, bool isAPM, BlobRequestOptions options)
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
byte[] buffer = GetRandomBuffer(size);
CloudBlockBlob blob = container.GetBlockBlobReference("blockblob");
blob.StreamWriteSizeInBytes = (int)(4 * Constants.MB);
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = new DictionaryKeyResolver();
resolver.Add(aesKey);
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
options.EncryptionPolicy = uploadPolicy;
OperationContext opContext = new OperationContext();
int uploadCount = 0;
opContext.SendingRequest += (sender, e) => uploadCount++;
using (MemoryStream stream = streamSeekable ? new MemoryStream(buffer) : new NonSeekableMemoryStream(buffer))
{
if (isAPM)
{
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
blob.EndUploadFromStream(blob.BeginUploadFromStream(
stream, size, null, options, opContext, ar => waitHandle.Set(), null));
}
}
else
{
blob.UploadFromStream(stream, size, null, options, opContext);
}
// Ensure that the user stream is open if it's seekable.
if (streamSeekable)
{
Assert.IsTrue(stream.CanSeek);
}
}
// Download the encrypted blob.
// Create the decryption policy to be used for download. There is no need to specify the
// key when the policy is only going to be used for downloads. Resolver is sufficient.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
// Download and decrypt the encrypted contents from the blob.
MemoryStream outputStream = new MemoryStream();
blob.DownloadToStream(outputStream, null, downloadOptions, null);
// Ensure that the user stream is open.
outputStream.Seek(0, SeekOrigin.Begin);
// Compare that the decrypted contents match the input data.
byte[] outputArray = outputStream.ToArray();
TestHelper.AssertBuffersAreEqualUptoIndex(outputArray, buffer, size - 1);
Assert.AreEqual(targetUploadOperations, uploadCount, "Incorrect number of operations in encrypted blob upload.");
}
finally
{
container.DeleteIfExists();
}
}
static void Main(string[] args)
{
Console.WriteLine("Blob encryption sample");
// Retrieve storage account information from connection string
// How to create a storage connection string - https://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/
CloudStorageAccount storageAccount = EncryptionShared.Utility.CreateStorageAccountFromConnectionString();
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(DemoContainer + Guid.NewGuid().ToString("N"));
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer = new byte[size];
Random rand = new Random();
rand.NextBytes(buffer);
CloudBlockBlob blob = container.GetBlockBlobReference("blockblob");
// Create the IKey used for encryption.
RsaKey key = new RsaKey("private:key1");
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(key, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
Console.WriteLine("Uploading the encrypted blob.");
// Upload the encrypted contents to the blob.
using (MemoryStream stream = new MemoryStream(buffer))
{
blob.UploadFromStream(stream, size, null, uploadOptions, null);
}
// Download the encrypted blob.
// For downloads, a resolver can be set up that will help pick the key based on the key id.
LocalResolver resolver = new LocalResolver();
resolver.Add(key);
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
Console.WriteLine("Downloading the encrypted blob.");
// Download and decrypt the encrypted contents from the blob.
using (MemoryStream outputStream = new MemoryStream())
{
blob.DownloadToStream(outputStream, null, downloadOptions, null);
}
Console.WriteLine("Press enter key to exit");
Console.ReadLine();
}
finally
{
container.DeleteIfExists();
}
}
public void CloudBlobEncryptionWithByteArray()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer = GetRandomBuffer(size);
byte[] outputBuffer = new byte[size];
CloudBlockBlob blob = container.GetBlockBlobReference("blockblob");
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = new DictionaryKeyResolver();
resolver.Add(aesKey);
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
// Upload the encrypted contents to the blob.
blob.UploadFromByteArray(buffer, 0, buffer.Length, null, uploadOptions, null);
// Download the encrypted blob.
// Create the decryption policy to be used for download. There is no need to specify the
// key when the policy is only going to be used for downloads. Resolver is sufficient.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
// Download and decrypt the encrypted contents from the blob.
blob.DownloadToByteArray(outputBuffer, 0, null, downloadOptions, null);
// Compare that the decrypted contents match the input data.
TestHelper.AssertBuffersAreEqual(buffer, outputBuffer);
}
finally
{
container.DeleteIfExists();
}
}
public void CloudBlobEncryptionWithFile()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer = GetRandomBuffer(size);
CloudBlockBlob blob = container.GetBlockBlobReference("blockblob");
// Create the Key to be used for wrapping.
SymmetricKey aesKey = new SymmetricKey("symencryptionkey");
// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = new DictionaryKeyResolver();
resolver.Add(aesKey);
// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions() { EncryptionPolicy = uploadPolicy };
string inputFileName = Path.GetTempFileName();
string outputFileName = Path.GetTempFileName();
using (FileStream file = new FileStream(inputFileName, FileMode.Create, FileAccess.Write))
{
file.Write(buffer, 0, buffer.Length);
}
// Upload the encrypted contents to the blob.
blob.UploadFromFile(inputFileName, FileMode.Open, null, uploadOptions, null);
// Download the encrypted blob.
// Create the decryption policy to be used for download. There is no need to specify the
// key when the policy is only going to be used for downloads. Resolver is sufficient.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions() { EncryptionPolicy = downloadPolicy };
// Download and decrypt the encrypted contents from the blob.
blob.DownloadToFile(outputFileName, FileMode.Create, null, downloadOptions, null);
// Compare that the decrypted contents match the input data.
using (FileStream inputFileStream = new FileStream(inputFileName, FileMode.Open, FileAccess.Read),
outputFileStream = new FileStream(outputFileName, FileMode.Open, FileAccess.Read))
{
TestHelper.AssertStreamsAreEqual(inputFileStream, outputFileStream);
}
}
finally
{
container.DeleteIfExists();
}
}
static void Main(string[] args)
{
Console.WriteLine("Blob encryption with Key Vault integration demonstrating key rotation from Key 1 to Key 2");
Console.WriteLine();
// Create two secrets and obtain their IDs. This is normally a one-time setup step.
// Although it is possible to use keys (rather than secrets) stored in Key Vault, this prevents caching.
// Therefore it is recommended to use secrets along with a caching resolver (see below).
string keyID1 = EncryptionShared.KeyVaultUtility.SetUpKeyVaultSecret("KeyRotationSampleSecret1");
string keyID2 = EncryptionShared.KeyVaultUtility.SetUpKeyVaultSecret("KeyRotationSampleSecret2");
// Retrieve storage account information from connection string
// How to create a storage connection string - https://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/
CloudStorageAccount storageAccount = EncryptionShared.Utility.CreateStorageAccountFromConnectionString();
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(DemoContainer + Guid.NewGuid().ToString("N"));
// Construct a resolver capable of looking up keys and secrets stored in Key Vault.
KeyVaultKeyResolver cloudResolver = new KeyVaultKeyResolver(EncryptionShared.KeyVaultUtility.GetAccessToken);
// Set up a caching resolver so the secrets can be cached on the client. This is the recommended usage
// pattern since the throttling targets for Storage and Key Vault services are orders of magnitude
// different.
CachingKeyResolver cachingResolver = new CachingKeyResolver(2, cloudResolver);
// Create key instances corresponding to the key IDs. This will cache the secrets.
IKey cloudKey1 = cachingResolver.ResolveKeyAsync(keyID1, CancellationToken.None).GetAwaiter().GetResult();
IKey cloudKey2 = cachingResolver.ResolveKeyAsync(keyID2, CancellationToken.None).GetAwaiter().GetResult();
// We begin with cloudKey1, and a resolver capable of resolving and caching Key Vault secrets.
BlobEncryptionPolicy encryptionPolicy = new BlobEncryptionPolicy(cloudKey1, cachingResolver);
client.DefaultRequestOptions.EncryptionPolicy = encryptionPolicy;
client.DefaultRequestOptions.RequireEncryption = true;
try
{
container.Create();
int size = 5 * 1024 * 1024;
byte[] buffer1 = new byte[size];
byte[] buffer2 = new byte[size];
Random rand = new Random();
rand.NextBytes(buffer1);
rand.NextBytes(buffer2);
// Upload the first blob using the secret stored in Azure Key Vault.
CloudBlockBlob blob = container.GetBlockBlobReference("blockblob1");
Console.WriteLine("Uploading Blob 1 using Key 1.");
// Upload the encrypted contents to the first blob.
using (MemoryStream stream = new MemoryStream(buffer1))
{
blob.UploadFromStream(stream, size);
}
Console.WriteLine("Downloading and decrypting Blob 1.");
// Download and decrypt the encrypted contents from the first blob.
using (MemoryStream outputStream = new MemoryStream())
{
blob.DownloadToStream(outputStream);
}
// At this point we will rotate our keys so new encrypted content will use the
// second key. Note that the same resolver is used, as this resolver is capable
// of decrypting blobs encrypted using either key.
Console.WriteLine("Rotating the active encryption key to Key 2.");
client.DefaultRequestOptions.EncryptionPolicy = new BlobEncryptionPolicy(cloudKey2, cachingResolver);
// Upload the second blob using the key stored in Azure Key Vault.
CloudBlockBlob blob2 = container.GetBlockBlobReference("blockblob2");
Console.WriteLine("Uploading Blob 2 using Key 2.");
// Upload the encrypted contents to the second blob.
using (MemoryStream stream = new MemoryStream(buffer2))
{
blob2.UploadFromStream(stream, size);
}
Console.WriteLine("Downloading and decrypting Blob 2.");
// Download and decrypt the encrypted contents from the second blob.
using (MemoryStream outputStream = new MemoryStream())
{
blob2.DownloadToStream(outputStream);
}
// Here we download and re-upload the first blob. This has the effect of updating
// the blob to use the new key.
using (MemoryStream memoryStream = new MemoryStream())
{
Console.WriteLine("Downloading and decrypting Blob 1.");
blob.DownloadToStream(memoryStream);
memoryStream.Seek(0, SeekOrigin.Begin);
Console.WriteLine("Re-uploading Blob 1 using Key 2.");
blob.UploadFromStream(memoryStream);
}
// For the purposes of demonstration, we now override the encryption policy to only recognize key 2.
BlobEncryptionPolicy key2OnlyPolicy = new BlobEncryptionPolicy(cloudKey2, null);
BlobRequestOptions key2OnlyOptions = new BlobRequestOptions()
{
EncryptionPolicy = key2OnlyPolicy
};
Console.WriteLine("Downloading and decrypting Blob 1.");
// The first blob can still be decrypted because it is using the second key.
using (MemoryStream outputStream = new MemoryStream())
{
blob.DownloadToStream(outputStream, options: key2OnlyOptions);
}
Console.WriteLine("Press enter key to exit");
Console.ReadLine();
}
finally
{
container.DeleteIfExists();
}
}
