private static void AddProperties(Blob blob, BlobItemProperties properties)
{
blob.TryAddProperties(
"CustomerProvidedKeySha256", properties.CustomerProvidedKeySha256,
"IncrementalCopy", properties.IncrementalCopy,
"ServerEncrypted", properties.ServerEncrypted,
"DeletedOn", properties.DeletedOn,
"RemainingRetentionDays", properties.RemainingRetentionDays,
"AccessTier", properties.AccessTier,
"AccessTierChangedOn", properties.AccessTierChangedOn,
"AccessTierInferred", properties.AccessTierInferred,
"ArchiveStatus", properties.ArchiveStatus,
"BlobSequenceNumber", properties.BlobSequenceNumber,
"BlobType", properties.BlobType,
"CacheControl", properties.CacheControl,
"ContentDisposition", properties.ContentDisposition,
"ContentEncoding", properties.ContentEncoding,
"ContentHash", properties.ContentHash.ToHexString(),
"ContentLanguage", properties.ContentLanguage,
"ContentLength", properties.ContentLength,
"ContentType", properties.ContentType,
"CopyCompletedOn", properties.CopyCompletedOn,
"CopyId", properties.CopyId,
"CopyProgress", properties.CopyProgress,
"CopySource", properties.CopySource,
"CopyStatus", properties.CopyStatus,
"CopyStatusDescription", properties.CopyStatusDescription,
"CreatedOn", properties.CreatedOn,
"DestinationSnapshot", properties.DestinationSnapshot,
"ETag", properties.ETag,
"LastModified", properties.LastModified,
"LeaseDuration", properties.LeaseDuration,
"LeaseState", properties.LeaseState,
"LeaseStatus", properties.LeaseStatus);
}
private string CreateBlobAndUploadToContainer(Mock <BlobContainerClient> containerMock, List <BlobItem> blobItems, string blobContent = "test", DateTimeOffset lastModified = default)
{
string blobName = Path.GetRandomFileName().Replace(".", "");
Mock <BlobBaseClient> item = new Mock <BlobBaseClient>();
if (lastModified == default)
{
lastModified = DateTimeOffset.UtcNow;
}
var blobProperties = BlobsModelFactory.BlobProperties(lastModified: lastModified);
item.Setup(x => x.GetPropertiesAsync(null, It.IsAny <CancellationToken>())).ReturnsAsync(Response.FromValue(blobProperties, null));
item.Setup(x => x.Name).Returns(blobName);
BlobItemProperties blobItemProperties = BlobsModelFactory.BlobItemProperties(true, lastModified: lastModified);
BlobItem blobItem = BlobsModelFactory.BlobItem(
name: blobName,
properties: blobItemProperties
);
blobItems.Add(blobItem);
Mock <BlobClient> blobClientMock = new Mock <BlobClient>();
blobClientMock.Setup(x => x.Name).Returns(blobName);
blobClientMock.Setup(x => x.Download(It.IsAny <CancellationToken>())).Returns(() =>
Response.FromValue(BlobsModelFactory.BlobDownloadInfo(content: new MemoryStream(Encoding.UTF8.GetBytes(blobContent))), null));
blobClientMock.Setup(x => x.GetProperties(It.IsAny <BlobRequestConditions>(), It.IsAny <CancellationToken>()))
.Returns(Response.FromValue(blobProperties, null));
containerMock.Setup(x => x.GetBlobClient(blobName)).Returns(blobClientMock.Object);
return(blobName);
}
public async Task <BlobItemProperties> GetPropertiesAsync(string name)
{
var blob = this.Container.GetBlobReference(name);
BlobItemProperties properties = null;
if (await blob.ExistsAsync())
{
await blob.FetchAttributesAsync();
properties = new BlobItemProperties(blob.Properties.Length, blob.Properties.ETag, blob.Properties.LastModified);
}
else
{
properties = BlobItemProperties.NotExists;
}
return(properties);
}
public async Task JobFinishedHandleAsyncShouldReturnTrueAndNotLogWhenNoErrors()
{
// Arrange
this.eventJobFinishedToPublish.Data = JObject.FromObject(this.outputJobFinishedData);
outputJobFinishedData.CorrelationData.TryGetValue("outputAssetContainer", out var containerUri);
EventGridEvent publishedEvent = null;
string blobName = "covfefe";
BlobItemProperties blobProps = BlobsModelFactory.BlobItemProperties(true);
var blobArray = new BlobItem[]
{
BlobsModelFactory.BlobItem(blobName, false, blobProps)
};
var blobsList = new List <BlobItem>(blobArray);
// Arrange Mocks
Mock.Get(storageService)
.Setup(x => x.ListBlobsAsync(It.IsAny <Uri>(), It.IsAny <StorageClientProviderContext>()))
.ReturnsAsync(blobsList);
Mock.Get(this.eventGridPublisher)
.Setup(x => x.PublishEventToTopic(It.IsAny <EventGridEvent>()))
.Callback <EventGridEvent>((ege) => publishedEvent = ege)
.ReturnsAsync(true);
// Act
var handleAsyncResult = await this.handler.HandleAsync(this.eventJobFinishedToPublish).ConfigureAwait(false);
// Assert
handleAsyncResult.ShouldBe(true, "handleAsync should always return true");
publishedEvent.Data.ShouldBeOfType <ResponseEncodeSuccessDTO>();
var responseEncodeSuccessData = (ResponseEncodeSuccessDTO)publishedEvent.Data;
for (int i = 0; i < blobArray.Length; i++)
{
responseEncodeSuccessData.Outputs[i].BlobUri.ShouldBe(new BlobUriBuilder(new Uri(containerUri))
{
BlobName = blobArray[i].Name
}.ToString());
}
}
public BlobInfo(BlobItemProperties blobItemProperties)
{
if (blobItemProperties.ContentLength.HasValue)
{
Size = blobItemProperties.ContentLength.Value;
}
else
{
Size = -1;
}
ContentMD5 = Convert.ToBase64String(blobItemProperties.ContentHash);
if (blobItemProperties.LastModified.HasValue)
{
LastModified = blobItemProperties.LastModified.Value;
}
else
{
LastModified = DateTimeOffset.MinValue;
}
}
public static string GetContentHash(this BlobItemProperties self)
{
return(Convert.ToBase64String(self.ContentHash));
}
private static File GetFileInfo(BlobClient blob, BlobSasBuilder readPermissions, BlobItemProperties blobItemProperties)
{
File file = new File
{
Length = blobItemProperties.ContentLength.GetValueOrDefault(),
ContentType = blobItemProperties.ContentType,
LastModified = blobItemProperties.LastModified.GetValueOrDefault().DateTime
};
FillFileInfo(file, blob, readPermissions);
return(file);
}
static int SasKeyExpiryTime = 1; // Default 1 hour
static async Task Main(string[] args)
{
Console.WriteLine("Starting Blob operations");
if (String.IsNullOrEmpty(AccountKey))
{
Console.WriteLine("The environment variable: AZURE_STORAGE_ACCOUNT_KEY, is not set!");
Environment.Exit(-1);
}
;
if (SourceTier.Equals(TargetTier))
{
Console.WriteLine("Source and Target Access Tiers cannot be the same!");
Environment.Exit(-1);
}
;
Stopwatch stopWatch = new Stopwatch();
try
{
// Start the stop watch
stopWatch.Start();
BlobServiceClient blobSvcClient = SharedAccessSignatureAuthAsync();
BlobContainerClient contClient = blobSvcClient.GetBlobContainerClient(ContainerName);
Uri[] blobUrls = new Uri[BatchSize];
int batchCount = 0;
int totalProcessed = 0;
List <Task <Azure.Response[]> > taskList = new List <Task <Azure.Response[]> >();
BlobClient blobClient = null;
IDictionary <string, string> dict = null;
Console.WriteLine("Enumerating blobs ...");
await foreach (BlobItem blobItem in contClient.GetBlobsAsync(BlobTraits.Metadata))
{
Console.WriteLine("-------------------------");
Console.WriteLine("\tName: " + blobItem.Name);
blobClient = contClient.GetBlobClient(blobItem.Name);
Console.WriteLine("\tUri: " + blobClient.Uri);
BlobItemProperties props = blobItem.Properties;
AccessTier?atier = props.AccessTier;
Console.WriteLine("\tAccess Tier: " + atier);
if (atier.Equals(SourceTier))
{
blobUrls[batchCount] = blobClient.Uri;
batchCount++;
}
;
dict = blobItem.Metadata;
if ((dict != null) && (dict.Count > 0))
{
foreach (KeyValuePair <string, string> item in dict)
{
Console.WriteLine("Key: {0}, Value: {1}", item.Key, item.Value);
}
}
else
{
Console.WriteLine("\tNo metadata for this blob item");
}
if (batchCount == BatchSize)
{
BlobBatchClient batch = blobSvcClient.GetBlobBatchClient();
taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier));
totalProcessed += batchCount;
batchCount = 0;
Console.WriteLine("-------------------------");
Console.WriteLine($"No. of Blobs moved to {TargetTier} tier: {totalProcessed}");
if (taskList.Count >= ConcurrentTasks)
{
// Wait for existing tasks to finish before proceeeding.
// This is to avoid out of resources issue.
Task.WaitAll(taskList.ToArray());
taskList.Clear();
}
}
;
}
if (batchCount > 0)
{
BlobBatchClient batch = blobSvcClient.GetBlobBatchClient();
taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier));
totalProcessed += batchCount;
}
;
if (taskList.Count > 0)
{
Task.WaitAll(taskList.ToArray()); // Wait for all Tasks to finish!
}
// Stop the timer
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}",
ts.Hours, ts.Minutes, ts.Seconds);
Console.WriteLine("------------------------------------");
Console.WriteLine($"Moved {totalProcessed} Blobs from {SourceTier} to {TargetTier} tier.");
Console.WriteLine($"Total Runtime: {elapsedTime}");
Console.WriteLine("------------------------------------");
}
catch (Exception ex)
{
Console.WriteLine("Encountered exception: " + ex);
}
}
