/// <summary>
/// set blob properties
/// </summary>
/// <param name="azureBlob">CloudBlob object</param>
/// <param name="meta">blob properties hashtable</param>
private async Task SetBlobProperties(IStorageBlobManagement localChannel, StorageBlob.CloudBlob blob, Hashtable properties)
{
if (properties == null)
{
return;
}
foreach (DictionaryEntry entry in properties)
{
string key = entry.Key.ToString();
string value = entry.Value.ToString();
Action <StorageBlob.BlobProperties, string> action = validCloudBlobProperties[key];
if (action != null)
{
action(blob.Properties, value);
}
}
AccessCondition accessCondition = null;
StorageBlob.BlobRequestOptions requestOptions = RequestOptions;
await Channel.SetBlobPropertiesAsync(blob, accessCondition, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false);
}
/// <summary>
/// set blob meta
/// </summary>
/// <param name="azureBlob">CloudBlob object</param>
/// <param name="meta">meta data hashtable</param>
private async Task SetBlobMeta(IStorageBlobManagement localChannel, StorageBlob.CloudBlob blob, Hashtable meta)
{
if (meta == null)
{
return;
}
foreach (DictionaryEntry entry in meta)
{
string key = entry.Key.ToString();
string value = entry.Value.ToString();
if (blob.Metadata.ContainsKey(key))
{
blob.Metadata[key] = value;
}
else
{
blob.Metadata.Add(key, value);
}
}
AccessCondition accessCondition = null;
StorageBlob.BlobRequestOptions requestOptions = RequestOptions;
await Channel.SetBlobMetadataAsync(blob, accessCondition, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false);
}
/// <summary>
/// On Task run successfully
/// </summary>
/// <param name="data">User data</param>
protected override void OnTaskSuccessful(DataMovementUserData data)
{
StorageBlob.CloudBlob blob = data.Data as StorageBlob.CloudBlob;
IStorageBlobManagement localChannel = data.Channel;
if (blob != null)
{
AccessCondition accessCondition = null;
StorageBlob.BlobRequestOptions requestOptions = RequestOptions;
if (this.pageBlobTier != null || this.standardBlobTier != null)
{
this.SetBlobTier(localChannel, blob, pageBlobTier, standardBlobTier).Wait();
}
try
{
localChannel.FetchBlobAttributesAsync(blob, accessCondition, requestOptions, OperationContext, CmdletCancellationToken).Wait();
}
catch (AggregateException e)
{
StorageException storageException = e.InnerException as StorageException;
//Handle the limited read permission.
if (storageException == null || !storageException.IsNotFoundException())
{
throw e.InnerException;
}
}
WriteCloudBlobObject(data.TaskId, localChannel, blob);
}
}
/// <summary>
/// set blob AccessTier
/// </summary>
/// <param name="azureBlob">CloudBlob object</param>
/// <param name="blockBlobTier">Block Blob Tier</param>
/// <param name="pageBlobTier">Page Blob Tier</param>
private async Task SetBlobTier(IStorageBlobManagement localChannel, StorageBlob.CloudBlob blob, PremiumPageBlobTier?pageBlobTier)
{
if (pageBlobTier == null)
{
return;
}
StorageBlob.BlobRequestOptions requestOptions = RequestOptions;
// The Blob Type and Blob Tier must match, since already checked they are match at the begin of ExecuteCmdlet().
if (pageBlobTier != null)
{
await Channel.SetPageBlobTierAsync((CloudPageBlob)blob, pageBlobTier.Value, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false);
}
}
public static void UploadFromFile(this StorageBlob.CloudBlob cloudBlob,
string path,
AccessCondition accessCondition = null,
StorageBlob.BlobRequestOptions options = null,
OperationContext operationContext = null)
{
if (StorageBlob.BlobType.BlockBlob == cloudBlob.BlobType)
{
(cloudBlob as StorageBlob.CloudBlockBlob).UploadFromFile(path, accessCondition, options, operationContext);
}
else if (StorageBlob.BlobType.PageBlob == cloudBlob.BlobType)
{
(cloudBlob as StorageBlob.CloudPageBlob).UploadFromFile(path, accessCondition, options, operationContext);
}
else if (StorageBlob.BlobType.AppendBlob == cloudBlob.BlobType)
{
(cloudBlob as StorageBlob.CloudAppendBlob).UploadFromFile(path, accessCondition, options, operationContext);
}
else
{
throw new InvalidOperationException(string.Format("Invalid blob type: {0}", cloudBlob.BlobType));
}
}
public static string DownloadText(CloudBlob blob, Encoding encoding, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
using (MemoryStream stream = new MemoryStream())
{
blob.DownloadToStream(stream, accessCondition, options, operationContext);
return(encoding.GetString(stream.ToArray()));
}
}
public virtual Task <BlobResultSegment> ListBlobsSegmentedAsync(bool useFlatBlobListing, BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
return(this.Container.ListBlobsSegmentedAsync(this.Prefix, useFlatBlobListing, blobListingDetails, maxResults, currentToken, options, operationContext, cancellationToken));
}
public virtual IEnumerable <IListBlobItem> ListBlobs(bool useFlatBlobListing = false, BlobListingDetails blobListingDetails = BlobListingDetails.None, BlobRequestOptions options = null, OperationContext operationContext = null)
{
return(this.Container.ListBlobs(this.Prefix, useFlatBlobListing, blobListingDetails, options, operationContext));
}
public async Task CloudAppendBlobAppendBlockSnapshotEncryptionScopeTask()
{
// Arrange
CloudBlobContainer container = GetRandomContainerReference();
await container.CreateAsync();
var encryptionScope = TestBase.TargetTenantConfig.EncryptionScope;
var options = new BlobRequestOptions
{
EncryptionScope = encryptionScope
};
var appendBlob = container.GetAppendBlobReference(GetRandomBlobName());
await appendBlob.CreateOrReplaceAsync(null, options, null);
try
{
byte[] buffer = GetRandomBuffer(1024);
using (MemoryStream sourceStream = new MemoryStream(buffer))
{
var context = new OperationContext();
// Act
await appendBlob.AppendBlockAsync(sourceStream, null, null, options, context);
// Assert
Assert.AreEqual(options.EncryptionScope, context.RequestResults.First().EncryptionScope);
// Arrange
context = new OperationContext();
using (MemoryStream downloadedBlob = new MemoryStream())
{
// Act
await appendBlob.DownloadRangeToStreamAsync(downloadedBlob, 0, 1024, null, options, context);
// Assert
TestHelper.AssertStreamsAreEqual(sourceStream, downloadedBlob);
Assert.AreEqual(options.EncryptionScope, context.RequestResults.First().EncryptionScope);
Assert.AreEqual(options.EncryptionScope, appendBlob.Properties.EncryptionScope);
}
// Arrange
context = new OperationContext();
var metadata = new Dictionary <string, string>
{
{ "foo", "bar" }
};
// Act
await appendBlob.SnapshotAsync(metadata, null, options, context);
// Assert
Assert.AreEqual(options.EncryptionScope, context.RequestResults.First().EncryptionScope);
Assert.AreEqual(options.EncryptionScope, appendBlob.Properties.EncryptionScope);
}
}
finally
{
await container.DeleteAsync();
}
}
public static string DownloadTextTask(CloudBlob blob, Encoding encoding, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
using (MemoryStream stream = new MemoryStream())
{
try
{
blob.DownloadToStreamAsync(stream, accessCondition, options, operationContext).Wait();
}
catch (AggregateException ex)
{
if (ex.InnerException != null)
{
throw ex.InnerException;
}
throw;
}
return(encoding.GetString(stream.ToArray()));
}
}
/// <summary>
/// Initializes a new instance of the BlobReadStreamBase class.
/// </summary>
/// <param name="blob">Blob reference to read from</param>
/// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the condition that must be met in order for the request to proceed. If <c>null</c>, no condition is used.</param>
/// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object that represents the context for the current operation.</param>
protected BlobReadStreamBase(CloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
if (options.ChecksumOptions.UseTransactionalMD5.Value)
{
CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", blob.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentMD5Size);
}
if (options.ChecksumOptions.UseTransactionalCRC64.Value)
{
CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", blob.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentCRC64Size);
}
this.blob = blob;
this.blobProperties = new BlobProperties(blob.Properties);
this.currentOffset = 0;
this.streamMinimumReadSizeInBytes = this.blob.StreamMinimumReadSizeInBytes;
this.internalBuffer = new MultiBufferMemoryStream(blob.ServiceClient.BufferManager);
this.accessCondition = accessCondition;
this.options = options;
this.operationContext = operationContext;
this.blobChecksum =
new ChecksumWrapper(
calcMd5: !(this.options.ChecksumOptions.DisableContentMD5Validation.Value || string.IsNullOrEmpty(this.blobProperties.ContentChecksum.MD5)),
calcCrc64: !(this.options.ChecksumOptions.DisableContentCRC64Validation.Value || string.IsNullOrEmpty(this.blobProperties.ContentChecksum.CRC64))
);
this.lastException = null;
}
public virtual ICancellableAsyncResult BeginListBlobsSegmented(bool useFlatBlobListing, BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, OperationContext operationContext, AsyncCallback callback, object state)
{
return(CancellableAsyncResultTaskWrapper.Create(token => this.ListBlobsSegmentedAsync(useFlatBlobListing, blobListingDetails, maxResults, currentToken, options, operationContext, token), callback, state));
}
public static string DownloadTextAPM(CloudBlob blob, Encoding encoding, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
using (MemoryStream stream = new MemoryStream())
{
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
IAsyncResult result = blob.BeginDownloadToStream(stream, accessCondition, options, operationContext, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blob.EndDownloadToStream(result);
return(encoding.GetString(stream.ToArray()));
}
}
}
public static void UploadTextTask(CloudBlob blob, string text, Encoding encoding, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
byte[] textAsBytes = encoding.GetBytes(text);
using (MemoryStream stream = new MemoryStream())
{
stream.Write(textAsBytes, 0, textAsBytes.Length);
if (blob.BlobType == BlobType.PageBlob)
{
int lastPageSize = (int)(stream.Length % 512);
if (lastPageSize != 0)
{
byte[] padding = new byte[512 - lastPageSize];
stream.Write(padding, 0, padding.Length);
}
}
stream.Seek(0, SeekOrigin.Begin);
blob.ServiceClient.DefaultRequestOptions.ParallelOperationThreadCount = 2;
try
{
if (blob.BlobType == BlobType.AppendBlob)
{
CloudAppendBlob blob1 = blob as CloudAppendBlob;
blob1.CreateOrReplaceAsync().Wait();
blob1.AppendBlock(stream, null);
}
else if (blob.BlobType == BlobType.PageBlob)
{
CloudPageBlob pageBlob = blob as CloudPageBlob;
pageBlob.UploadFromStreamAsync(stream, accessCondition, options, operationContext).Wait();
}
else
{
CloudBlockBlob blockBlob = blob as CloudBlockBlob;
blockBlob.UploadFromStreamAsync(stream, accessCondition, options, operationContext).Wait();
}
}
catch (AggregateException ex)
{
if (ex.InnerException != null)
{
throw ex.InnerException;
}
throw;
}
}
}
public static void UploadTextAPM(CloudBlob blob, string text, Encoding encoding, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
byte[] textAsBytes = encoding.GetBytes(text);
using (MemoryStream stream = new MemoryStream())
{
stream.Write(textAsBytes, 0, textAsBytes.Length);
if (blob.BlobType == BlobType.PageBlob)
{
int lastPageSize = (int)(stream.Length % 512);
if (lastPageSize != 0)
{
byte[] padding = new byte[512 - lastPageSize];
stream.Write(padding, 0, padding.Length);
}
}
stream.Seek(0, SeekOrigin.Begin);
blob.ServiceClient.DefaultRequestOptions.ParallelOperationThreadCount = 2;
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
if (blob.BlobType == BlobType.AppendBlob)
{
CloudAppendBlob blob1 = blob as CloudAppendBlob;
IAsyncResult result = blob1.BeginCreateOrReplace(
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
blob1.EndCreateOrReplace(result);
result = blob1.BeginAppendBlock(stream, null,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
blob1.EndAppendBlock(result);
}
else if (blob.BlobType == BlobType.PageBlob)
{
CloudPageBlob pageBlob = blob as CloudPageBlob;
IAsyncResult result = pageBlob.BeginUploadFromStream(stream, accessCondition, options, operationContext,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
pageBlob.EndUploadFromStream(result);
}
else
{
CloudBlockBlob blockBlob = blob as CloudBlockBlob;
IAsyncResult result = blockBlob.BeginUploadFromStream(stream, accessCondition, options, operationContext,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
blockBlob.EndUploadFromStream(result);
}
}
}
}
internal static BlobRequestOptions ApplyDefaults(BlobRequestOptions options, BlobType blobType, CloudBlobClient serviceClient, bool applyExpiry = true)
{
BlobRequestOptions modifiedOptions = new BlobRequestOptions(options);
modifiedOptions.RetryPolicy =
modifiedOptions.RetryPolicy
?? serviceClient.DefaultRequestOptions.RetryPolicy
?? BaseDefaultRequestOptions.RetryPolicy;
modifiedOptions.AbsorbConditionalErrorsOnRetry =
modifiedOptions.AbsorbConditionalErrorsOnRetry
?? serviceClient.DefaultRequestOptions.AbsorbConditionalErrorsOnRetry
?? BaseDefaultRequestOptions.AbsorbConditionalErrorsOnRetry;
#if !(WINDOWS_RT || NETCORE)
modifiedOptions.EncryptionPolicy =
modifiedOptions.EncryptionPolicy
?? serviceClient.DefaultRequestOptions.EncryptionPolicy
?? BaseDefaultRequestOptions.EncryptionPolicy;
modifiedOptions.RequireEncryption =
modifiedOptions.RequireEncryption
?? serviceClient.DefaultRequestOptions.RequireEncryption
?? BaseDefaultRequestOptions.RequireEncryption;
#endif
modifiedOptions.LocationMode =
modifiedOptions.LocationMode
?? serviceClient.DefaultRequestOptions.LocationMode
?? BaseDefaultRequestOptions.LocationMode;
modifiedOptions.ServerTimeout =
modifiedOptions.ServerTimeout
?? serviceClient.DefaultRequestOptions.ServerTimeout
?? BaseDefaultRequestOptions.ServerTimeout;
modifiedOptions.MaximumExecutionTime =
modifiedOptions.MaximumExecutionTime
?? serviceClient.DefaultRequestOptions.MaximumExecutionTime
?? BaseDefaultRequestOptions.MaximumExecutionTime;
modifiedOptions.NetworkTimeout =
modifiedOptions.NetworkTimeout
?? serviceClient.DefaultRequestOptions.NetworkTimeout
?? BaseDefaultRequestOptions.NetworkTimeout;
modifiedOptions.ParallelOperationThreadCount =
modifiedOptions.ParallelOperationThreadCount
?? serviceClient.DefaultRequestOptions.ParallelOperationThreadCount
?? BaseDefaultRequestOptions.ParallelOperationThreadCount;
modifiedOptions.SingleBlobUploadThresholdInBytes =
modifiedOptions.SingleBlobUploadThresholdInBytes
?? serviceClient.DefaultRequestOptions.SingleBlobUploadThresholdInBytes
?? BaseDefaultRequestOptions.SingleBlobUploadThresholdInBytes;
if (applyExpiry && !modifiedOptions.OperationExpiryTime.HasValue && modifiedOptions.MaximumExecutionTime.HasValue)
{
modifiedOptions.OperationExpiryTime = DateTime.Now + modifiedOptions.MaximumExecutionTime.Value;
}
#if (WINDOWS_PHONE && WINDOWS_DESKTOP)
modifiedOptions.ChecksumOptions.DisableContentMD5Validation = BaseDefaultRequestOptions.DisableContentMD5Validation;
modifiedOptions.ChecksumOptions.StoreContentMD5 = BaseDefaultRequestOptions.ChecksumOptions.StoreBlobContentMD5;
modifiedOptions.ChecksumOptions.UseTransactionalMD5 = BaseDefaultRequestOptions.UseTransactionalMD5;
modifiedOptions.ChecksumOptions.DisableContentCRC64Validation = BaseDefaultRequestOptions.DisableContentCRC64Validation;
modifiedOptions.ChecksumOptions.StoreContentCRC64 = BaseDefaultRequestOptions.ChecksumOptions.StoreBlobContentCRC64;
modifiedOptions.ChecksumOptions.UseTransactionalCRC64 = BaseDefaultRequestOptions.UseTransactionalCRC64;
#else
modifiedOptions.ChecksumOptions.DisableContentMD5Validation =
modifiedOptions.ChecksumOptions.DisableContentMD5Validation
?? serviceClient.DefaultRequestOptions.ChecksumOptions.DisableContentMD5Validation
?? BaseDefaultRequestOptions.ChecksumOptions.DisableContentMD5Validation;
modifiedOptions.ChecksumOptions.StoreContentMD5 =
modifiedOptions.ChecksumOptions.StoreContentMD5
?? serviceClient.DefaultRequestOptions.ChecksumOptions.StoreContentMD5
?? (blobType == BlobType.BlockBlob); // must be computed
modifiedOptions.ChecksumOptions.UseTransactionalMD5 =
modifiedOptions.ChecksumOptions.UseTransactionalMD5
?? serviceClient.DefaultRequestOptions.ChecksumOptions.UseTransactionalMD5
?? BaseDefaultRequestOptions.ChecksumOptions.UseTransactionalMD5;
modifiedOptions.ChecksumOptions.DisableContentCRC64Validation =
modifiedOptions.ChecksumOptions.DisableContentCRC64Validation
?? serviceClient.DefaultRequestOptions.ChecksumOptions.DisableContentCRC64Validation
?? BaseDefaultRequestOptions.ChecksumOptions.DisableContentCRC64Validation;
modifiedOptions.ChecksumOptions.StoreContentCRC64 =
modifiedOptions.ChecksumOptions.StoreContentCRC64
?? serviceClient.DefaultRequestOptions.ChecksumOptions.StoreContentCRC64
?? (blobType == BlobType.BlockBlob); // must be computed
modifiedOptions.ChecksumOptions.UseTransactionalCRC64 =
modifiedOptions.ChecksumOptions.UseTransactionalCRC64
?? serviceClient.DefaultRequestOptions.ChecksumOptions.UseTransactionalCRC64
?? BaseDefaultRequestOptions.ChecksumOptions.UseTransactionalCRC64;
#endif
modifiedOptions.CustomerProvidedKey = options?.CustomerProvidedKey;
return(modifiedOptions);
}
public async override Task DownloadRangeToStreamAsync(Stream target, long?offset, long?length, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
Console.WriteLine("Called with offset = " + offset + ", length = " + length);
MemoryStream str = new MemoryStream();
await base.DownloadRangeToStreamAsync(str, offset, length, accessCondition, options, operationContext, cancellationToken);
str.Position = 0;
byte[] buffer = new byte[writeSize];
while (str.Position < str.Length)
{
cancellationToken.ThrowIfCancellationRequested();
int copied = await str.ReadAsync(buffer, 0, buffer.Length);
await target.WriteAsync(buffer, 0, copied);
if (str.Position < str.Length)
{
await Task.Delay(delayPerWriteInMs);
}
}
}
public virtual BlobResultSegment ListBlobsSegmented(bool useFlatBlobListing, BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, OperationContext operationContext)
{
return(this.Container.ListBlobsSegmented(this.Prefix, useFlatBlobListing, blobListingDetails, maxResults, currentToken, options, operationContext));
}