public async Task CreateAsync()
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
var blobName = GetNewBlobName();
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName));
// Act
Response <BlobContentInfo> response = await blob.CreateAsync();
// Assert
Assert.IsNotNull(response.GetRawResponse().Headers.RequestId);
IList <BlobItem> blobs = await test.Container.GetBlobsAsync().ToListAsync();
Assert.AreEqual(1, blobs.Count);
Assert.AreEqual(blobName, blobs.First().Name);
}
public async Task CreateAsync_AccessConditionsFail()
{
var garbageLeaseId = GetGarbageLeaseId();
AccessConditionParameters[] data = new[]
{
new AccessConditionParameters {
IfModifiedSince = NewDate
},
new AccessConditionParameters {
IfUnmodifiedSince = OldDate
},
new AccessConditionParameters {
Match = GarbageETag
},
new AccessConditionParameters {
NoneMatch = ReceivedETag
},
new AccessConditionParameters {
LeaseId = garbageLeaseId
}
};
foreach (AccessConditionParameters parameters in data)
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
// AppendBlob needs to exists for us to test CreateAsync() with access conditions
await blob.CreateAsync();
parameters.NoneMatch = await SetupBlobMatchCondition(blob, parameters.NoneMatch);
AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions(
parameters: parameters,
lease: true);
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blob.CreateAsync(conditions: accessConditions),
e => { });
}
}
public async Task AppendBlobSample()
{
// Instantiate a new BlobServiceClient using a connection string.
BlobServiceClient blobServiceClient = new BlobServiceClient(TestConfigurations.DefaultTargetTenant.ConnectionString);
// Instantiate a new BlobContainerClient
BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient("mycontainer4");
try
{
// Create new Container in the Service
await blobContainerClient.CreateAsync();
// Instantiate a new PageBlobClient
AppendBlobClient appendBlobClient = blobContainerClient.GetAppendBlobClient("appendblob");
// Create PageBlob in the Service
await appendBlobClient.CreateAsync();
// Append content to AppendBlob
using (FileStream fileStream = File.OpenRead("Samples/SampleSource.txt"))
{
await appendBlobClient.AppendBlockAsync(fileStream);
}
// Download PageBlob
using (FileStream fileStream = File.Create("AppendDestination.txt"))
{
Response <BlobDownloadInfo> downloadResponse = await appendBlobClient.DownloadAsync();
await downloadResponse.Value.Content.CopyToAsync(fileStream);
}
// Delete PageBlob in the Service
await appendBlobClient.DeleteAsync();
}
finally
{
// Delete Container in the Service
await blobContainerClient.DeleteAsync();
}
}
public AppendBlobWriteStream(
AppendBlobClient appendBlobClient,
long bufferSize,
long position,
AppendBlobRequestConditions conditions,
IProgress <long> progressHandler
// TODO #27253
//UploadTransactionalHashingOptions hashingOptions
) : base(
position,
bufferSize,
progressHandler
// TODO #27253
//hashingOptions
)
{
ValidateBufferSize(bufferSize);
_appendBlobClient = appendBlobClient;
_conditions = conditions ?? new AppendBlobRequestConditions();
}
public async Task AppendBlockAsync_EncryptionScope()
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
var blobName = GetNewBlobName();
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName));
blob = InstrumentClient(blob.WithEncryptionScope(TestConfigDefault.EncryptionScope));
var data = GetRandomBuffer(Constants.KB);
await blob.CreateAsync();
// Act
using var stream = new MemoryStream(data);
Response <BlobAppendInfo> response = await blob.AppendBlockAsync(
content : stream);
// Assert
Assert.AreEqual(TestConfigDefault.EncryptionScope, response.Value.EncryptionScope);
}
public async Task AppendBlockAsync_MD5Fail()
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
await blob.CreateAsync();
var data = GetRandomBuffer(Constants.KB);
// Act
using (var stream = new MemoryStream(data))
{
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blob.AppendBlockAsync(
content: stream,
transactionalContentHash: MD5.Create().ComputeHash(Encoding.UTF8.GetBytes("garbage"))),
e => Assert.AreEqual("Md5Mismatch", e.ErrorCode.Split('\n')[0]));
}
}
public async Task FindBlobsByTagAsync_AccountSas(AccountSasPermissions accountSasPermissions)
{
// Arrange
BlobServiceClient service = GetServiceClient_SharedKey();
await using DisposingContainer test = await GetTestContainerAsync();
string blobName = GetNewBlobName();
AppendBlobClient appendBlob = InstrumentClient(test.Container.GetAppendBlobClient(blobName));
string tagKey = "myTagKey";
string tagValue = "myTagValue";
Dictionary <string, string> tags = new Dictionary <string, string>
{
{ tagKey, tagValue }
};
AppendBlobCreateOptions options = new AppendBlobCreateOptions
{
Tags = tags
};
await appendBlob.CreateAsync(options);
string expression = $"\"{tagKey}\"='{tagValue}'";
// It takes a few seconds for Filter Blobs to pick up new changes
await Delay(2000);
// Act
SasQueryParameters sasQueryParameters = GetNewAccountSas(permissions: accountSasPermissions);
BlobServiceClient sasServiceClient = new BlobServiceClient(new Uri($"{service.Uri}?{sasQueryParameters}"), GetOptions());
List <TaggedBlobItem> blobs = new List <TaggedBlobItem>();
await foreach (Page <TaggedBlobItem> page in sasServiceClient.FindBlobsByTagsAsync(expression).AsPages())
{
blobs.AddRange(page.Values);
}
// Assert
TaggedBlobItem filterBlob = blobs.Where(r => r.BlobName == blobName).FirstOrDefault();
Assert.IsNotNull(filterBlob);
}
public async Task AppendBlockAsync_CPK()
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
var blobName = GetNewBlobName();
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName));
CustomerProvidedKey customerProvidedKey = GetCustomerProvidedKey();
blob = InstrumentClient(blob.WithCustomerProvidedKey(customerProvidedKey));
var data = GetRandomBuffer(Constants.KB);
await blob.CreateAsync();
// Act
using var stream = new MemoryStream(data);
Response <BlobAppendInfo> response = await blob.AppendBlockAsync(
content : stream);
// Assert
Assert.AreEqual(customerProvidedKey.EncryptionKeyHash, response.Value.EncryptionKeySha256);
}
public async Task AppendBlockAsync_MD5()
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
await blob.CreateAsync();
var data = GetRandomBuffer(Constants.KB);
// Act
using (var stream = new MemoryStream(data))
{
Response <BlobAppendInfo> response = await blob.AppendBlockAsync(
content : stream,
transactionalContentHash : MD5.Create().ComputeHash(data));
// Assert
Assert.IsNotNull(response.GetRawResponse().Headers.RequestId);
}
}
private async Task GetAppendBlobClientAsync(BlobServiceClient blobServiceClient, string blobContainerName, string blobName, bool bypassBlobCreationValidation, long?blobSizeLimitBytes = null)
{
//try to get a reference to a AppendBlobClient which is below the max blocks threshold.
for (int i = currentBlobRollSequence; i < 999; i++)
{
string rolledBlobName = GetRolledBlobName(blobName, i);
AppendBlobClient newAppendBlobClient = await GetBlobReferenceAsync(blobServiceClient, blobContainerName, rolledBlobName, bypassBlobCreationValidation);
var blobPropertiesResponse = await newAppendBlobClient.GetPropertiesAsync();
var blobProperties = blobPropertiesResponse.Value;
if (ValidateBlobProperties(blobProperties, blobSizeLimitBytes))
{
currentAppendBlobClient = newAppendBlobClient;
currentBlobName = blobName;
currentBlobRollSequence = i;
break;
}
}
}
public void Ctor_ConnectionString()
{
var accountName = "accountName";
var accountKey = Convert.ToBase64String(new byte[] { 0, 1, 2, 3, 4, 5 });
var credentials = new StorageSharedKeyCredential(accountName, accountKey);
var blobEndpoint = new Uri("http://127.0.0.1/" + accountName);
var blobSecondaryEndpoint = new Uri("http://127.0.0.1/" + accountName + "-secondary");
var connectionString = new StorageConnectionString(credentials, blobStorageUri: (blobEndpoint, blobSecondaryEndpoint));
var containerName = GetNewContainerName();
var blobName = GetNewBlobName();
AppendBlobClient blob = InstrumentClient(new AppendBlobClient(connectionString.ToString(true), containerName, blobName, GetOptions()));
var builder = new BlobUriBuilder(blob.Uri);
Assert.AreEqual(containerName, builder.BlobContainerName);
Assert.AreEqual(blobName, builder.BlobName);
Assert.AreEqual("accountName", builder.AccountName);
}
public async Task GetAppendBlobClient_NonAsciiName()
{
//Arrange
await using DisposingContainer test = await GetTestContainerAsync();
string blobName = GetNewNonAsciiBlobName();
//Act
AppendBlobClient blob = test.Container.GetAppendBlobClient(blobName);
await blob.CreateAsync();
//Assert
List <string> names = new List <string>();
await foreach (BlobItem pathItem in test.Container.GetBlobsAsync())
{
names.Add(pathItem.Name);
}
// Verify the file name exists in the filesystem
Assert.AreEqual(1, names.Count);
Assert.Contains(blobName, names);
}
public async Task BlobVersionIdentitySas_AllPermissions()
{
// Arrange
BlobServiceClient oauthService = GetServiceClient_OauthAccount();
string containerName = GetNewContainerName();
string blobName = GetNewBlobName();
await using DisposingContainer test = await GetTestContainerAsync(containerName : containerName, service : oauthService);
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName));
Response <BlobContentInfo> createResponse = await blob.CreateAsync();
IDictionary <string, string> metadata = BuildMetadata();
Response <BlobInfo> metadataResponse = await blob.SetMetadataAsync(metadata);
BlobSasBuilder blobSasBuilder = new BlobSasBuilder()
{
ExpiresOn = Recording.UtcNow.AddDays(1),
BlobContainerName = test.Container.Name,
BlobName = blobName,
BlobVersionId = createResponse.Value.VersionId
};
blobSasBuilder.SetPermissions(BlobVersionSasPermissions.All);
BlobUriBuilder blobUriBuilder = new BlobUriBuilder(blob.Uri)
{
VersionId = createResponse.Value.VersionId,
Sas = blobSasBuilder.ToSasQueryParameters(userDelegationKey.Value, oauthService.AccountName)
};
// Act
AppendBlobClient sasBlobClient = InstrumentClient(new AppendBlobClient(blobUriBuilder.ToUri(), GetOptions()));
await sasBlobClient.DeleteAsync();
}
public async Task BlobSnapshotIdentitySas_AllPermissions()
{
// Arrange
BlobServiceClient oauthService = GetServiceClient_OauthAccount();
string containerName = GetNewContainerName();
string blobName = GetNewBlobName();
await using DisposingContainer test = await GetTestContainerAsync(containerName : containerName, service : oauthService);
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName));
await blob.CreateAsync();
Response <BlobSnapshotInfo> snapshotResponse = await blob.CreateSnapshotAsync();
BlobSasBuilder blobSasBuilder = new BlobSasBuilder()
{
ExpiresOn = Recording.UtcNow.AddDays(1),
BlobContainerName = test.Container.Name,
BlobName = blobName,
Snapshot = snapshotResponse.Value.Snapshot
};
blobSasBuilder.SetPermissions(SnapshotSasPermissions.All);
BlobUriBuilder blobUriBuilder = new BlobUriBuilder(blob.Uri)
{
Snapshot = snapshotResponse.Value.Snapshot,
Sas = blobSasBuilder.ToSasQueryParameters(userDelegationKey.Value, oauthService.AccountName)
};
// Act
AppendBlobClient sasBlobClient = InstrumentClient(new AppendBlobClient(blobUriBuilder.ToUri(), GetOptions()));
await sasBlobClient.GetPropertiesAsync();
}
public async Task <AppendBlobClient> GetBlobReferenceAsync(BlobServiceClient blobServiceClient, string blobContainerName, string blobName, bool bypassBlobCreationValidation)
{
var blobContainer = blobServiceClient.GetBlobContainerClient(blobContainerName);
await CreateBlobContainerIfNotExistsAsync(blobContainer, bypassBlobCreationValidation).ConfigureAwait(false);
AppendBlobClient newAppendBlobClient = null;
try
{
newAppendBlobClient = blobContainer.GetAppendBlobClient(blobName);
// TODO-VPL: CreateOrReplaceAsync does not exist in the new SDK
// TODO-VPL: AccessCondition is nowhere to be seen... here is the original line:
//newAppendBlobClient.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null).GetAwaiter().GetResult();
await newAppendBlobClient.CreateIfNotExistsAsync();
}
catch (RequestFailedException ex) when(ex.Status == (int)HttpStatusCode.Conflict && ex.ErrorCode == "BlobAlreadyExists")
{
//StorageException (http 409 conflict, error code BlobAlreadyExists) is thrown due to the AccessCondition. The append blob already exists.
//No problem this is expected
}
catch (Exception ex)
{
Debugging.SelfLog.WriteLine($"Failed to create blob: {ex}");
throw;
}
// TODO-VPL: This is done differently in the new SDK ; we need to do a get properties and they return the properties, i.e. done elsewhere
//if (newAppendBlobClient != null)
//{
// //this is the first time the code gets its hands on this blob reference, get the blob properties from azure.
// //used later on to know when to roll over the file if the 50.000 max blocks is getting close.
// await newAppendBlobClient.FetchAttributesAsync().ConfigureAwait(false);
//}
return(newAppendBlobClient);
}
public async Task AppendBlockFromUriAsync_Min()
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
await test.Container.SetAccessPolicyAsync(PublicAccessType.BlobContainer);
var data = GetRandomBuffer(Constants.KB);
using (var stream = new MemoryStream(data))
{
AppendBlobClient sourceBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
await sourceBlob.CreateAsync();
await sourceBlob.AppendBlockAsync(stream);
AppendBlobClient destBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
await destBlob.CreateAsync();
// Act
await destBlob.AppendBlockFromUriAsync(sourceBlob.Uri, new HttpRange(0, Constants.KB));
}
}
public async Task AppendBlockAsync_ProgressReporting()
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
var blobName = GetNewBlobName();
AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName));
await blob.CreateAsync();
const int blobSize = 4 * Constants.MB;
var data = GetRandomBuffer(blobSize);
TestProgress progress = new TestProgress();
// Act
using (var stream = new MemoryStream(data))
{
await blob.AppendBlockAsync(stream, progressHandler : progress);
}
// Assert
Assert.IsFalse(progress.List.Count == 0);
Assert.AreEqual(blobSize, progress.List[progress.List.Count - 1]);
}
private async Task WriteLogAsync()
{
try
{
BlobContainerClient blobContainerClient = await CreateContainer();
AppendBlobClient appendBlobClient = await CreateBlob(blobContainerClient);
using (var ms = new MemoryStream())
{
string message = "";
TextWriter tw = new StreamWriter(ms, Encoding.UTF8, appendBlobClient.AppendBlobMaxAppendBlockBytes);
while (!_logMessages.IsEmpty)
{
_logMessages.TryDequeue(out message);
tw.Write(message);
}
tw.Flush();
ms.Position = 0;
appendBlobClient.AppendBlock(ms);
}
}
catch (Exception ex) { throw ex; }
}
public async Task ContainerSas_AllPermissions()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
string blobName = GetNewBlobName();
BlobSasBuilder blobSasBuilder = new BlobSasBuilder(
permissions: BlobContainerSasPermissions.All,
expiresOn: Recording.UtcNow.AddDays(1))
{
BlobContainerName = test.Container.Name,
};
BlobUriBuilder blobUriBuilder = new BlobUriBuilder(test.Container.Uri)
{
BlobName = blobName,
Sas = blobSasBuilder.ToSasQueryParameters(GetNewSharedKeyCredentials())
};
// Act
AppendBlobClient appendBlobClient = InstrumentClient(new AppendBlobClient(blobUriBuilder.ToUri(), GetOptions()));
await appendBlobClient.CreateAsync();
}
private void SetCloudBlobBlockCount(AppendBlobClient appendBlobClient, int newBlockCount)
{
// TODO-VPL: I do not know Fake Fx enough to fx that one... I disabled compilation of the entire file
appendBlobClient.Properties.GetType().GetProperty(nameof(BlobProperties.AppendBlobCommittedBlockCount)).SetValue(appendBlobClient.Properties, newBlockCount, null);
}
public async Task AppendBlockFromUriAsync_AccessConditionsFail()
{
var garbageLeaseId = GetGarbageLeaseId();
AccessConditionParameters[] testCases = new[]
{
new AccessConditionParameters {
IfModifiedSince = NewDate
},
new AccessConditionParameters {
IfUnmodifiedSince = OldDate
},
new AccessConditionParameters {
Match = GarbageETag
},
new AccessConditionParameters {
NoneMatch = ReceivedETag
},
new AccessConditionParameters {
LeaseId = garbageLeaseId
},
new AccessConditionParameters {
AppendPosE = 1
},
new AccessConditionParameters {
MaxSizeLTE = 1
},
new AccessConditionParameters {
SourceIfModifiedSince = NewDate
},
new AccessConditionParameters {
SourceIfUnmodifiedSince = OldDate
},
new AccessConditionParameters {
SourceIfMatch = GarbageETag
},
new AccessConditionParameters {
SourceIfNoneMatch = ReceivedETag
}
};
foreach (AccessConditionParameters parameters in testCases)
{
await using DisposingContainer test = await GetTestContainerAsync();
// Arrange
await test.Container.SetAccessPolicyAsync(PublicAccessType.BlobContainer);
var data = GetRandomBuffer(7);
using (var stream = new MemoryStream(data))
{
AppendBlobClient sourceBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
await sourceBlob.CreateAsync();
await sourceBlob.AppendBlockAsync(stream);
AppendBlobClient destBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName()));
await destBlob.CreateAsync();
parameters.NoneMatch = await SetupBlobMatchCondition(destBlob, parameters.NoneMatch);
parameters.SourceIfNoneMatch = await SetupBlobMatchCondition(sourceBlob, parameters.SourceIfNoneMatch);
AppendBlobRequestConditions accessConditions = BuildDestinationAccessConditions(
parameters: parameters,
lease: true,
appendPosAndMaxSize: true);
AppendBlobRequestConditions sourceAccessConditions = BuildSourceAccessConditions(parameters);
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
destBlob.AppendBlockFromUriAsync(
sourceUri: sourceBlob.Uri,
conditions: accessConditions,
sourceConditions: sourceAccessConditions),
actualException => Assert.IsTrue(true)
);
}
}
}
public AzureBlobsLogWriter(BlobContainerClient blobsContainerClient,
string fileName,
bool appendOpen = false)
{
fileName = AzureBlobsLogsInterface.PathFixer(fileName);
_blobsContainerClient = blobsContainerClient;
_logClient = _blobsContainerClient.GetAppendBlobClient(fileName);
ETag currentETag;
if (_previousOpenAttempts.ContainsKey(fileName) && appendOpen)
{
// We've opened this blob before and want to be non-destructive. We don't need to CreateIfNotExists, which could be VERY slow.
currentETag = _logClient.GetProperties().Value.ETag;
}
else
{
try
{
// Create the file non-destructively if needed, guaranteeing write continuity on creation by grabbing the etag of the create, if needed
if (appendOpen)
{
var response = _logClient.CreateIfNotExists();
if (response != null)
{
currentETag = response.Value.ETag;
}
else
{
currentETag = _logClient.GetProperties().Value.ETag;
}
}
else
{
currentETag = _logClient.Create().Value.ETag;
}
}
catch { currentETag = _logClient.GetProperties().Value.ETag; }
}
// Try to grab the blob lease
_leaseClient = _logClient.GetBlobLeaseClient();
// The blob hasn't be touched since the last time. This is a candidate for breaking the lease.
if (_previousOpenAttempts.ContainsKey(fileName) && (_previousOpenAttempts[fileName].ToString().Equals(currentETag.ToString())))
{
_previousOpenAttempts[fileName] = currentETag;
// The blob hasn't been updated. Try to break the lease and reacquire
var requestConditions = new BlobRequestConditions();
requestConditions = new BlobRequestConditions();
requestConditions.IfMatch = currentETag;
// If the condition fails in the break, it's because someone else managed to touch the file, so give up
ETag newETag;
try
{
newETag = _leaseClient.Break(null, requestConditions).Value.ETag;
}
catch (Exception e) { newETag = currentETag; }
var etagCondition = new RequestConditions();
etagCondition.IfMatch = newETag;
// If the condition fails, someone snuck in and grabbed the lock before we could. Give up.
_curLease = _leaseClient.Acquire(TimeSpan.FromSeconds(-1), etagCondition).Value;
}
else
{
// Not a candidate for breaking the lease. Just try to acquire.
_previousOpenAttempts[fileName] = currentETag;
_curLease = _leaseClient.Acquire(TimeSpan.FromSeconds(-1)).Value;
}
_leaseCondition = new AppendBlobRequestConditions();
_leaseCondition.LeaseId = _curLease.LeaseId;
// We got the lease! Set up thread to periodically touch the blob to prevent others from breaking the lease.
_blobMetadata = _logClient.GetProperties().Value.Metadata;
_stopRelockThread = false;
_relockThreadStopped = false;
_leaseRenewThread = new Thread(() =>
{
while (!_stopRelockThread)
{
Thread.Sleep(100);
var response = _logClient.SetMetadata(_blobMetadata, _leaseCondition);
}
_relockThreadStopped = true;
})
{
IsBackground = true
};
_leaseRenewThread.Start();
_bytesToSend = new MemoryStream();
Debug.Assert(_logClient.Exists());
}
public static BlobBaseClient GetTrack2BlobClient(Uri blobUri, AzureStorageContext context, BlobClientOptions options = null, global::Azure.Storage.Blobs.Models.BlobType?blobType = null)
{
BlobBaseClient blobClient;
if (options is null)
{
options = new BlobClientOptions();
}
if (context.StorageAccount.Credentials.IsToken) //Oauth
{
if (blobType == null)
{
blobClient = new BlobBaseClient(blobUri, context.Track2OauthToken, options);
}
else
{
switch (blobType.Value)
{
case global::Azure.Storage.Blobs.Models.BlobType.Page:
blobClient = new PageBlobClient(blobUri, context.Track2OauthToken, options);
break;
case global::Azure.Storage.Blobs.Models.BlobType.Append:
blobClient = new AppendBlobClient(blobUri, context.Track2OauthToken, options);
break;
default: //Block
blobClient = new BlockBlobClient(blobUri, context.Track2OauthToken, options);
break;
}
}
}
else if (context.StorageAccount.Credentials.IsSharedKey) //Shared Key
{
if (blobType == null)
{
blobClient = new BlobBaseClient(blobUri, new StorageSharedKeyCredential(context.StorageAccountName, context.StorageAccount.Credentials.ExportBase64EncodedKey()), options);
}
else
{
switch (blobType.Value)
{
case global::Azure.Storage.Blobs.Models.BlobType.Page:
blobClient = new PageBlobClient(blobUri, new StorageSharedKeyCredential(context.StorageAccountName, context.StorageAccount.Credentials.ExportBase64EncodedKey()), options);
break;
case global::Azure.Storage.Blobs.Models.BlobType.Append:
blobClient = new AppendBlobClient(blobUri, new StorageSharedKeyCredential(context.StorageAccountName, context.StorageAccount.Credentials.ExportBase64EncodedKey()), options);
break;
default: //Block
blobClient = new BlockBlobClient(blobUri, new StorageSharedKeyCredential(context.StorageAccountName, context.StorageAccount.Credentials.ExportBase64EncodedKey()), options);
break;
}
}
}
else //SAS or Anonymous
{
if (blobType == null)
{
blobClient = new BlobBaseClient(blobUri, options);
}
else
{
switch (blobType.Value)
{
case global::Azure.Storage.Blobs.Models.BlobType.Page:
blobClient = new PageBlobClient(blobUri, options);
break;
case global::Azure.Storage.Blobs.Models.BlobType.Append:
blobClient = new AppendBlobClient(blobUri, options);
break;
default: //Block
blobClient = new BlockBlobClient(blobUri, options);
break;
}
}
}
return(blobClient);
}
public static void Call([BlobTrigger(BlobPath)] AppendBlobClient blob)
{
TaskSource.TrySetResult(blob);
}
public static AppendBlobClient WithCustomerProvidedKey(
this AppendBlobClient blob,
CustomerProvidedKey customerProvidedKey) =>
new AppendBlobClient(
ToHttps(blob.Uri),
BuildClientConfigurationWithCpk(blob.ClientConfiguration, customerProvidedKey));
//TODO remove ToHttps() after service fixes HTTPS bug.
public static AppendBlobClient WithEncryptionScope(
this AppendBlobClient blob,
string encryptionScope)
=> new AppendBlobClient(
ToHttps(blob.Uri),
BuildClientConfigurationWithEncryptionScope(blob.ClientConfiguration, encryptionScope));
public AppendBlobHandler(AppendBlobClient blobClient, AzureFileStorage azureFileStorage)
{
_blobClient = blobClient;
_azureFileStorage = azureFileStorage;
}
/// <summary>
/// Upload File to blob with storage Client library API
/// </summary>
internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagement localChannel, string filePath, StorageBlob.CloudBlob blob)
{
BlobClientOptions options = null;
if (this.Force.IsPresent ||
!blob.Exists() ||
ShouldContinue(string.Format(Resources.OverwriteConfirmation, blob.Uri), null))
{
// Prepare blob Properties, MetaData, accessTier
BlobHttpHeaders blobHttpHeaders = CreateBlobHttpHeaders(BlobProperties);
IDictionary <string, string> metadata = new Dictionary <string, string>();
SetBlobMeta_Track2(metadata, this.Metadata);
AccessTier?accesstier = GetAccessTier_Track2(this.standardBlobTier, this.pageBlobTier);
//Prepare progress handler
long fileSize = new FileInfo(ResolvedFileName).Length;
string activity = String.Format(Resources.SendAzureBlobActivity, this.File, blob.Name, blob.Container.Name);
string status = Resources.PrepareUploadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
IProgress <long> progressHandler = new Progress <long>((finishedBytes) =>
{
if (pr != null)
{
// Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
pr.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize);
pr.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, pr.PercentComplete);
Console.WriteLine(finishedBytes);
this.OutputStream.WriteProgress(pr);
}
});
using (FileStream stream = System.IO.File.OpenRead(ResolvedFileName))
{
//block blob
if (string.Equals(blobType, BlockBlobType, StringComparison.InvariantCultureIgnoreCase))
{
BlobClient blobClient = GetTrack2BlobClient(blob, localChannel.StorageContext, options);
StorageTransferOptions trasnferOption = new StorageTransferOptions()
{
MaximumConcurrency = this.GetCmdletConcurrency()
};
BlobUploadOptions uploadOptions = new BlobUploadOptions();
uploadOptions.Metadata = metadata;
uploadOptions.HttpHeaders = blobHttpHeaders;
uploadOptions.Conditions = this.BlobRequestConditions;
uploadOptions.AccessTier = accesstier;
uploadOptions.ProgressHandler = progressHandler;
uploadOptions.TransferOptions = trasnferOption;
await blobClient.UploadAsync(stream, uploadOptions, CmdletCancellationToken).ConfigureAwait(false);
}
//Page or append blob
else if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) ||
string.Equals(blobType, AppendBlobType, StringComparison.InvariantCultureIgnoreCase))
{
PageBlobClient pageblobClient = null;
AppendBlobClient appendblobClient = null;
//Create Blob
if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page
{
if (fileSize % 512 != 0)
{
throw new ArgumentException(String.Format("File size {0} Bytes is invalid for PageBlob, must be a multiple of 512 bytes.", fileSize.ToString()));
}
pageblobClient = GetTrack2PageBlobClient(blob, localChannel.StorageContext, options);
PageBlobCreateOptions createOptions = new PageBlobCreateOptions();
createOptions.Metadata = metadata;
createOptions.HttpHeaders = blobHttpHeaders;
createOptions.Conditions = this.PageBlobRequestConditions;
Response <BlobContentInfo> blobInfo = await pageblobClient.CreateAsync(fileSize, createOptions, CmdletCancellationToken).ConfigureAwait(false);
}
else //append
{
appendblobClient = GetTrack2AppendBlobClient(blob, localChannel.StorageContext, options);
AppendBlobCreateOptions createOptions = new AppendBlobCreateOptions();
createOptions.Metadata = metadata;
createOptions.HttpHeaders = blobHttpHeaders;
createOptions.Conditions = this.AppendBlobRequestConditions;
Response <BlobContentInfo> blobInfo = await appendblobClient.CreateAsync(createOptions, CmdletCancellationToken).ConfigureAwait(false);
}
// Upload blob content
byte[] uploadcache4MB = null;
byte[] uploadcache = null;
progressHandler.Report(0);
long offset = 0;
while (offset < fileSize)
{
// Get chunk size and prepare cache
int chunksize = size4MB;
if (chunksize <= (fileSize - offset)) // Chunk size will be 4MB
{
if (uploadcache4MB == null)
{
uploadcache4MB = new byte[size4MB];
}
uploadcache = uploadcache4MB;
}
else // last chunk can < 4MB
{
chunksize = (int)(fileSize - offset);
if (uploadcache4MB == null)
{
uploadcache = new byte[chunksize];
}
else
{
uploadcache = uploadcache4MB;
}
}
//Get content to upload for the chunk
int readoutcount = await stream.ReadAsync(uploadcache, 0, (int)chunksize).ConfigureAwait(false);
MemoryStream chunkContent = new MemoryStream(uploadcache, 0, readoutcount);
//Upload content
if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page
{
Response <PageInfo> pageInfo = await pageblobClient.UploadPagesAsync(chunkContent, offset, null, null, null, CmdletCancellationToken).ConfigureAwait(false);
}
else //append
{
Response <BlobAppendInfo> pageInfo = await appendblobClient.AppendBlockAsync(chunkContent, null, null, null, CmdletCancellationToken).ConfigureAwait(false);
}
// Update progress
offset += readoutcount;
progressHandler.Report(offset);
}
if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) && accesstier != null)
{
await pageblobClient.SetAccessTierAsync(accesstier.Value, cancellationToken : CmdletCancellationToken).ConfigureAwait(false);
}
}
else
{
throw new InvalidOperationException(string.Format(
CultureInfo.CurrentCulture,
Resources.InvalidBlobType,
blobType,
BlobName));
}
}
WriteCloudBlobObject(taskId, localChannel, blob);
}
}
public async Task InitializeAsync(WebJobsTestEnvironment testEnvironment)
{
RandomNameResolver nameResolver = new RandomNameResolver();
Host = new HostBuilder()
.ConfigureDefaultTestHost <BlobBindingEndToEndTests>(b =>
{
b.AddAzureStorageBlobs().AddAzureStorageQueues();
b.AddAzureStorageCoreServices();
})
.ConfigureServices(services =>
{
services.AddSingleton <INameResolver>(nameResolver);
})
.Build();
JobHost = Host.GetJobHost();
BlobServiceClient = new BlobServiceClient(testEnvironment.PrimaryStorageAccountConnectionString);
BlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(ContainerName));
Assert.False(await BlobContainer.ExistsAsync());
await BlobContainer.CreateAsync();
OutputBlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(OutputContainerName));
var pageBlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(PageBlobContainerName));
Assert.False(await pageBlobContainer.ExistsAsync());
await pageBlobContainer.CreateAsync();
var hierarchicalBlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(HierarchicalBlobContainerName));
Assert.False(await hierarchicalBlobContainer.ExistsAsync());
await hierarchicalBlobContainer.CreateAsync();
var appendBlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(AppendBlobContainerName));
Assert.False(await appendBlobContainer.ExistsAsync());
await appendBlobContainer.CreateAsync();
await Host.StartAsync();
// upload some test blobs
BlockBlobClient blob = BlobContainer.GetBlockBlobClient("blob1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("blob2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("blob3");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("file1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("file2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("overwrite");
await blob.UploadTextAsync(TestData);
// add a couple hierarchical blob paths
blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/blob1");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/blob2");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/sub/blob3");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobClient("blob4");
await blob.UploadTextAsync(TestData);
byte[] bytes = new byte[512];
byte[] testBytes = Encoding.UTF8.GetBytes(TestData);
for (int i = 0; i < testBytes.Length; i++)
{
bytes[i] = testBytes[i];
}
PageBlobClient pageBlob = pageBlobContainer.GetPageBlobClient("blob1");
await pageBlob.UploadFromByteArrayAsync(bytes, 0);
pageBlob = pageBlobContainer.GetPageBlobClient("blob2");
await pageBlob.UploadFromByteArrayAsync(bytes, 0);
AppendBlobClient appendBlob = appendBlobContainer.GetAppendBlobClient("blob1");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobClient("blob2");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobClient("blob3");
await appendBlob.UploadTextAsync(TestData);
}
protected void SendStream(AppendBlobClient blob, Stream stream)
{
blob.AppendBlock(stream);
}
