/// <summary>
/// Create a blob with specified blob type
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">Blob name</param>
/// <param name="type">Blob type</param>
/// <returns>ICloudBlob object</returns>
public ICloudBlob CreateBlob(CloudBlobContainer container, string blobName, Storage.BlobType type)
{
if (type == Microsoft.WindowsAzure.Storage.Blob.BlobType.BlockBlob)
{
return CreateBlockBlob(container, blobName);
}
else
{
return CreatePageBlob(container, blobName);
}
}
public static StorageBlob.ICloudBlob GetBlob(StorageBlob.CloudBlobContainer container, string blobName, StorageType blobType)
{
StorageBlob.ICloudBlob blob = null;
if (blobType == StorageType.BlockBlob)
{
blob = container.GetBlockBlobReference(blobName);
}
else
{
blob = container.GetPageBlobReference(blobName);
}
return blob;
}
public static void UploadFromFile(this StorageBlob.CloudBlob cloudBlob,
string path,
FileMode mode,
AccessCondition accessCondition = null,
StorageBlob.BlobRequestOptions options = null,
OperationContext operationContext = null)
{
if (StorageBlob.BlobType.BlockBlob == cloudBlob.BlobType)
{
(cloudBlob as StorageBlob.CloudBlockBlob).UploadFromFile(path, mode, accessCondition, options, operationContext);
}
else if (StorageBlob.BlobType.PageBlob == cloudBlob.BlobType)
{
(cloudBlob as StorageBlob.CloudPageBlob).UploadFromFile(path, mode, accessCondition, options, operationContext);
}
else if (StorageBlob.BlobType.AppendBlob == cloudBlob.BlobType)
{
(cloudBlob as StorageBlob.CloudAppendBlob).UploadFromFile(path, mode, accessCondition, options, operationContext);
}
else
{
throw new InvalidOperationException(string.Format("Invalid blob type: {0}", cloudBlob.BlobType));
}
}
public static void PackBlobCompareData(StorageBlob.ICloudBlob blob, Dictionary<string, object> dic)
{
dic["Length"] = blob.Properties.Length;
dic["ContentType"] = blob.Properties.ContentType;
dic["LastModified"] = blob.Properties.LastModified;
dic["SnapshotTime"] = blob.SnapshotTime;
}
public static bool WaitForCopyOperationComplete(StorageBlob.ICloudBlob destBlob, int maxRetry = 100)
{
int retryCount = 0;
int sleepInterval = 1000; //ms
if (destBlob == null)
{
return false;
}
do
{
if (retryCount > 0)
{
Test.Info(String.Format("{0}th check current copy state and it's {1}. Wait for copy completion", retryCount, destBlob.CopyState.Status));
}
Thread.Sleep(sleepInterval);
destBlob.FetchAttributes();
retryCount++;
}
while (destBlob.CopyState.Status == StorageBlob.CopyStatus.Pending && retryCount < maxRetry);
Test.Info(String.Format("Final Copy status is {0}", destBlob.CopyState.Status));
return destBlob.CopyState.Status != StorageBlob.CopyStatus.Pending;
}
/// <summary>
/// set blob meta
/// </summary>
/// <param name="azureBlob">CloudBlob object</param>
/// <param name="meta">meta data hashtable</param>
private async Task SetBlobMeta(IStorageBlobManagement localChannel, StorageBlob.CloudBlob blob, Hashtable meta)
{
if (meta == null)
{
return;
}
foreach (DictionaryEntry entry in meta)
{
string key = entry.Key.ToString();
string value = entry.Value.ToString();
if (blob.Metadata.ContainsKey(key))
{
blob.Metadata[key] = value;
}
else
{
blob.Metadata.Add(key, value);
}
}
AccessCondition accessCondition = null;
StorageBlob.BlobRequestOptions requestOptions = RequestOptions;
await Channel.SetBlobMetadataAsync(blob, accessCondition, requestOptions, OperationContext, CmdletCancellationToken);
}
public StorageBlob.CloudBlobContainer CreateContainer(string containerName, StorageBlob.BlobContainerPublicAccessType permission)
{
StorageBlob.CloudBlobContainer container = CreateContainer(containerName);
StorageBlob.BlobContainerPermissions containerPermission = new StorageBlob.BlobContainerPermissions();
containerPermission.PublicAccess = permission;
container.SetPermissions(containerPermission);
return container;
}
public List<StorageBlob.ICloudBlob> CreateRandomBlob(StorageBlob.CloudBlobContainer container)
{
int count = random.Next(1, 5);
List<string> blobNames = new List<string>();
for (int i = 0; i < count; i++)
{
blobNames.Add(Utility.GenNameString("blob"));
}
return CreateRandomBlob(container, blobNames);
}
/// <summary>
/// Create a list of blobs with random properties/metadata/blob type
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">Blob name</param>
/// <returns>ICloudBlob object</returns>
public StorageBlob.ICloudBlob CreateRandomBlob(StorageBlob.CloudBlobContainer container, string blobName)
{
int switchKey = 0;
switchKey = random.Next(0, 2);
if (switchKey == 0)
{
return CreatePageBlob(container, blobName);
}
else
{
return CreateBlockBlob(container, blobName);
}
}
/// <summary>
/// generate random blob properties and metadata
/// </summary>
/// <param name="blob">ICloudBlob object</param>
private void GenerateBlobPropertiesAndMetaData(StorageBlob.ICloudBlob blob)
{
blob.Properties.ContentEncoding = Utility.GenNameString("encoding");
blob.Properties.ContentLanguage = Utility.GenNameString("lang");
int minMetaCount = 1;
int maxMetaCount = 5;
int minMetaValueLength = 10;
int maxMetaValueLength = 20;
int count = random.Next(minMetaCount, maxMetaCount);
for (int i = 0; i < count; i++)
{
string metaKey = Utility.GenNameString("metatest");
int valueLength = random.Next(minMetaValueLength, maxMetaValueLength);
string metaValue = Utility.GenNameString("metavalue-", valueLength);
blob.Metadata.Add(metaKey, metaValue);
}
blob.SetProperties();
blob.SetMetadata();
blob.FetchAttributes();
}
/// <summary>
/// create a list of blobs with random properties/metadata/blob type
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">a list of blob names</param>
/// <returns>a list of cloud page blobs</returns>
public List<StorageBlob.ICloudBlob> CreateRandomBlob(StorageBlob.CloudBlobContainer container, List<string> blobNames)
{
List<StorageBlob.ICloudBlob> blobs = new List<StorageBlob.ICloudBlob>();
foreach (string blobName in blobNames)
{
blobs.Add(CreateRandomBlob(container, blobName));
}
blobs = blobs.OrderBy(blob => blob.Name).ToList();
return blobs;
}
/// <summary>
/// create a block blob with random properties and metadata
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">Block blob name</param>
/// <returns>ICloudBlob object</returns>
public StorageBlob.ICloudBlob CreateBlockBlob(StorageBlob.CloudBlobContainer container, string blobName)
{
StorageBlob.CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
int maxBlobSize = 1024 * 1024;
string md5sum = string.Empty;
int blobSize = random.Next(maxBlobSize);
byte[] buffer = new byte[blobSize];
using (MemoryStream ms = new MemoryStream(buffer))
{
random.NextBytes(buffer);
//ms.Read(buffer, 0, buffer.Length);
blockBlob.UploadFromStream(ms);
md5sum = Convert.ToBase64String(Helper.GetMD5(buffer));
}
blockBlob.Properties.ContentMD5 = md5sum;
GenerateBlobPropertiesAndMetaData(blockBlob);
Test.Info(string.Format("create block blob '{0}' in container '{1}'", blobName, container.Name));
return blockBlob;
}
/// <summary>
/// create a new page blob with random properties and metadata
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">blob name</param>
/// <returns>ICloudBlob object</returns>
public StorageBlob.ICloudBlob CreatePageBlob(StorageBlob.CloudBlobContainer container, string blobName)
{
StorageBlob.CloudPageBlob pageBlob = container.GetPageBlobReference(blobName);
int size = random.Next(1, 10) * PageBlobUnitSize;
pageBlob.Create(size);
byte[] buffer = new byte[size];
string md5sum = Convert.ToBase64String(Helper.GetMD5(buffer));
pageBlob.Properties.ContentMD5 = md5sum;
GenerateBlobPropertiesAndMetaData(pageBlob);
Test.Info(string.Format("create page blob '{0}' in container '{1}'", blobName, container.Name));
return pageBlob;
}
/// <summary>
/// remove specified container
/// </summary>
/// <param name="Container">Cloud blob container object</param>
public void RemoveContainer(StorageBlob.CloudBlobContainer Container)
{
RemoveContainer(Container.Name);
}
/// <summary>
/// upload file to azure blob
/// </summary>
/// <param name="taskId">Task id</param>
/// <param name="filePath">local file path</param>
/// <param name="blob">destination azure blob object</param>
internal virtual async Task Upload2Blob(long taskId, IStorageBlobManagement localChannel, string filePath, StorageBlob.CloudBlob blob)
{
string activity = String.Format(Resources.SendAzureBlobActivity, filePath, blob.Name, blob.Container.Name);
string status = Resources.PrepareUploadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
FileInfo fileInfo = new FileInfo(filePath);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr,
TotalSize = fileInfo.Length
};
await DataMovementTransferHelper.DoTransfer(() =>
{
return this.TransferManager.UploadAsync(filePath,
blob,
null,
this.GetTransferContext(data),
this.CmdletCancellationToken);
},
data.Record,
this.OutputStream);
if (this.BlobProperties != null || this.BlobMetadata != null)
{
await TaskEx.WhenAll(
this.SetBlobProperties(localChannel, blob, this.BlobProperties),
this.SetBlobMeta(localChannel, blob, this.BlobMetadata));
}
try
{
await localChannel.FetchBlobAttributesAsync(
blob,
AccessCondition.GenerateEmptyCondition(),
this.RequestOptions,
this.OperationContext,
this.CmdletCancellationToken);
}
catch (StorageException e)
{
//Handle the limited read permission.
if (!e.IsNotFoundException())
{
throw;
}
}
WriteCloudBlobObject(data.TaskId, localChannel, blob);
}
/// <summary>
/// Create a snapshot for the specified ICloudBlob object
/// </summary>
/// <param name="blob">ICloudBlob object</param>
public StorageBlob.ICloudBlob SnapShot(StorageBlob.ICloudBlob blob)
{
StorageBlob.ICloudBlob snapshot = default(StorageBlob.ICloudBlob);
switch (blob.BlobType)
{
case StorageBlob.BlobType.BlockBlob:
snapshot = ((StorageBlob.CloudBlockBlob)blob).CreateSnapshot();
break;
case StorageBlob.BlobType.PageBlob:
snapshot = ((StorageBlob.CloudPageBlob)blob).CreateSnapshot();
break;
default:
throw new ArgumentException(string.Format("Unsupport blob type {0} when create snapshot", blob.BlobType));
}
Test.Info(string.Format("Create snapshot for '{0}' at {1}", blob.Name, snapshot.SnapshotTime));
return snapshot;
}
/// <summary>
/// set blob properties
/// </summary>
/// <param name="azureBlob">CloudBlob object</param>
/// <param name="meta">blob properties hashtable</param>
private async Task SetBlobProperties(IStorageBlobManagement localChannel, StorageBlob.CloudBlob blob, Hashtable properties)
{
if (properties == null)
{
return;
}
foreach (DictionaryEntry entry in properties)
{
string key = entry.Key.ToString();
string value = entry.Value.ToString();
Action<StorageBlob.BlobProperties, string> action = validCloudBlobProperties[key];
if (action != null)
{
action(blob.Properties, value);
}
}
AccessCondition accessCondition = null;
StorageBlob.BlobRequestOptions requestOptions = RequestOptions;
await Channel.SetBlobPropertiesAsync(blob, accessCondition, requestOptions, OperationContext, CmdletCancellationToken);
}
public static void PackContainerCompareData(StorageBlob.CloudBlobContainer container, Dictionary<string, object> dic)
{
StorageBlob.BlobContainerPermissions permissions = container.GetPermissions();
dic["PublicAccess"] = permissions.PublicAccess;
dic["Permission"] = permissions;
dic["LastModified"] = container.Properties.LastModified;
}
/// <summary>
/// Create a blob with specified blob type
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">Blob name</param>
/// <param name="type">Blob type</param>
/// <returns>ICloudBlob object</returns>
public ICloudBlob CreateBlob(CloudBlobContainer container, string blobName, StorageBlob.BlobType type)
{
if (type == StorageBlob.BlobType.BlockBlob)
{
return CreateBlockBlob(container, blobName);
}
else
{
return CreatePageBlob(container, blobName);
}
}
/// <summary>
/// upload file to azure blob
/// </summary>
/// <param name="taskId">Task id</param>
/// <param name="filePath">local file path</param>
/// <param name="blob">destination azure blob object</param>
internal virtual async Task Upload2Blob(long taskId, IStorageBlobManagement localChannel, string filePath, StorageBlob.CloudBlob blob)
{
string activity = String.Format(Resources.SendAzureBlobActivity, filePath, blob.Name, blob.Container.Name);
string status = Resources.PrepareUploadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr
};
TransferJob uploadJob =
new TransferJob(
new TransferLocation(filePath),
new TransferLocation(blob),
TransferMethod.SyncCopy);
await this.RunTransferJob(uploadJob, data);
if (this.BlobProperties != null || this.BlobMetadata != null)
{
await TaskEx.WhenAll(
this.SetBlobProperties(localChannel, blob, this.BlobProperties),
this.SetBlobMeta(localChannel, blob, this.BlobMetadata));
}
try
{
await localChannel.FetchBlobAttributesAsync(
blob,
AccessCondition.GenerateEmptyCondition(),
this.RequestOptions,
this.OperationContext,
this.CmdletCancellationToken);
}
catch (StorageException e)
{
//Handle the limited read permission.
if (!e.IsNotFoundException())
{
throw;
}
}
WriteCloudBlobObject(data.TaskId, localChannel, blob);
}
