private async Task AnonymizeBlobsInNdJsonFormat(ActivityInputData inputData, BlobContainerClient inputContainer, BlobContainerClient outputContainer, string inputBlobPrefix, string outputBlobPrefix)
{
await foreach (BlobItem blob in inputContainer.GetBlobsAsync(BlobTraits.None, BlobStates.None, inputBlobPrefix, default))
{
if (IsInputFileInJsonFormat(blob.Name))
{
continue;
}
string outputBlobName = GetOutputBlobName(blob.Name, inputBlobPrefix, outputBlobPrefix);
Console.WriteLine($"[{blob.Name}]：Processing... output to container '{outputContainer.Name}'");
var inputBlobClient = new BlobClient(inputData.SourceStorageConnectionString, inputContainer.Name, blob.Name, BlobClientOptions.Value);
var outputBlobClient = new BlockBlobClient(inputData.DestinationStorageConnectionString, outputContainer.Name, outputBlobName, BlobClientOptions.Value);
if (inputData.SkipExistedFile && await outputBlobClient.ExistsAsync().ConfigureAwait(false))
{
Console.WriteLine($"[{blob.Name}]：'{outputBlobName}' already exist. Skip");
}
else
{
await outputBlobClient.DeleteIfExistsAsync().ConfigureAwait(false);
await AnonymizeSingleBlobInNdJsonFormatAsync(inputBlobClient, outputBlobClient, blob.Name, inputBlobPrefix);
}
}
}
public async Task QueryAsync_IfTags()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
Dictionary <string, string> tags = new Dictionary <string, string>
{
{ "coolTag", "true" }
};
await blockBlobClient.SetTagsAsync(tags);
BlobQueryOptions blobQueryOptions = new BlobQueryOptions
{
Conditions = new BlobRequestConditions
{
TagConditions = "\"coolTag\" = 'true'"
}
};
// Act
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
querySqlExpression : query,
options : blobQueryOptions);
using StreamReader streamReader = new StreamReader(response.Value.Content);
string s = await streamReader.ReadToEndAsync();
// Assert
Assert.AreEqual("400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n", s);
}
public async Task QueryAsync_ParquetConfiguration()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
using FileStream stream = File.OpenRead(
$"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}Resources{Path.DirectorySeparatorChar}parquet.parquet");
await blockBlobClient.UploadAsync(stream);
// Act
string query = @"select * from blobstorage where id < 1;";
BlobQueryOptions options = new BlobQueryOptions
{
InputTextConfiguration = new BlobQueryParquetTextOptions()
};
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options : options);
// Assert
using StreamReader streamReader = new StreamReader(response.Value.Content);
string s = await streamReader.ReadToEndAsync();
// Assert
Assert.AreEqual("0,mdifjt55.ea3,mdifjt55.ea3\n", s);
}
public void DownloadBlockBlobStream()
{
// Get the connection string to storage account
string connectionString = ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString;
// Set the container name
string containerName = "testcontainer";
// Set the complete block blob path and name. You can also use just the name if you want to download from root
string blobName = "parentDirectory/childDirectory/testBlockBlob.txt";
// We are using the Blobs.Specialized.BlockBlobClient to quickly create block blob object with minimal code
BlockBlobClient blob = new BlockBlobClient(connectionString, containerName, blobName);
using (var memoryStream = new MemoryStream())
{
// Downloading block blob content to MemoryStream
blob.DownloadTo(memoryStream);
// Reading contents from memory stream
var content = Encoding.UTF8.GetString(memoryStream.ToArray());
// We can also Deserialize the content to an object
// var obj = Newtonsoft.Json.JsonConvert.DeserializeObject<MyObject>(content);
// Console.WriteLine($"ID: '{obj.ID}', Name: '{obj.Name}', SomeDate: '{obj.SomeDate}'");
Console.WriteLine(content);
}
}
public async Task QueryAsync_Progress()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
// Act
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
TestProgress progressReporter = new TestProgress();
BlobQueryOptions options = new BlobQueryOptions
{
ProgressHandler = progressReporter
};
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options);
using StreamReader streamReader = new StreamReader(response.Value.Content);
await streamReader.ReadToEndAsync();
Assert.AreEqual(2, progressReporter.List.Count);
Assert.AreEqual(Constants.KB, progressReporter.List[0]);
Assert.AreEqual(Constants.KB, progressReporter.List[1]);
}
/// <inheritdoc />
public async Task <Operation?> GetAsync(ITenant tenant, Guid operationId)
{
BlobContainerClient container = await this.containerSource.GetBlobContainerClientFromTenantAsync(
tenant,
OperationsV2ConfigKey,
OperationsV3ConfigKey,
ContainerName)
.ConfigureAwait(false);
BlockBlobClient blob = container.GetBlockBlobClient(GetBlobName(tenant, operationId));
Response <BlobDownloadResult> response;
try
{
response = await blob.DownloadContentAsync().ConfigureAwait(false);
}
catch (Azure.RequestFailedException rfx)
when(rfx.Status == 404)
{
return(null);
}
// Note: although BlobDownloadResult supports direct deserialization from JSON, using System.Text.Json
// (meaning it can work directly with UTF-8 content, avoiding the conversion to UTF-16 we're doing
// here) we currently depend on the JSON.NET serialization settings mechanism, so we have to use
// this more inefficient route for now.
string json = response.Value.Content.ToString();
return(JsonConvert.DeserializeObject <Operation>(json, this.serializerSettings));
}
/// <inheritdoc />
public async Task <DicomDataset> GetInstanceMetadataAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
BlockBlobClient cloudBlockBlob = GetInstanceBlockBlob(versionedInstanceIdentifier);
DicomDataset dicomDataset = null;
await ExecuteAsync(async() =>
{
await using (Stream stream = _recyclableMemoryStreamManager.GetStream(GetInstanceMetadataStreamTagName))
{
await cloudBlockBlob.DownloadToAsync(stream, cancellationToken);
stream.Seek(0, SeekOrigin.Begin);
using (var streamReader = new StreamReader(stream, MetadataEncoding))
using (var jsonTextReader = new JsonTextReader(streamReader))
{
dicomDataset = _jsonSerializer.Deserialize <DicomDataset>(jsonTextReader);
}
}
});
return(dicomDataset);
}
public async Task SyncCopyFromUri_ImmutableStorageWithVersioning()
{
// Arrange
await using DisposingImmutableStorageWithVersioningContainer vlwContainer = await GetTestVersionLevelWormContainer(TestConfigOAuth);
BlobBaseClient srcBlob = await GetNewBlobClient(vlwContainer.Container);
BlockBlobClient destBlob = InstrumentClient(vlwContainer.Container.GetBlockBlobClient(GetNewBlobName()));
BlobImmutabilityPolicy immutabilityPolicy = new BlobImmutabilityPolicy
{
ExpiresOn = Recording.UtcNow.AddMinutes(5),
PolicyMode = BlobImmutabilityPolicyMode.Unlocked
};
// The service rounds Immutability Policy Expiry to the nearest second.
DateTimeOffset expectedImmutabilityPolicyExpiry = RoundToNearestSecond(immutabilityPolicy.ExpiresOn.Value);
BlobCopyFromUriOptions options = new BlobCopyFromUriOptions
{
DestinationImmutabilityPolicy = immutabilityPolicy,
LegalHold = true
};
// Act
await destBlob.SyncCopyFromUriAsync(srcBlob.Uri, options);
// Assert
Response <BlobProperties> propertiesResponse = await destBlob.GetPropertiesAsync();
Assert.AreEqual(expectedImmutabilityPolicyExpiry, propertiesResponse.Value.ImmutabilityPolicy.ExpiresOn);
Assert.AreEqual(immutabilityPolicy.PolicyMode, propertiesResponse.Value.ImmutabilityPolicy.PolicyMode);
Assert.IsTrue(propertiesResponse.Value.HasLegalHold);
}
/// <inheritdoc />
public async Task <Uri> StoreFileAsync(
VersionedInstanceIdentifier versionedInstanceIdentifier,
Stream stream,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
EnsureArg.IsNotNull(stream, nameof(stream));
BlockBlobClient blob = GetInstanceBlockBlob(versionedInstanceIdentifier);
stream.Seek(0, SeekOrigin.Begin);
try
{
await blob.UploadAsync(
stream,
new BlobHttpHeaders()
{
ContentType = KnownContentTypes.ApplicationDicom,
},
metadata : null,
conditions : null,
accessTier : null,
progressHandler : null,
cancellationToken);
return(blob.Uri);
}
catch (Exception ex)
{
throw new DataStoreException(ex);
}
}
protected virtual async Task DeleteAsync(string fullPath, CancellationToken cancellationToken)
{
(BlobContainerClient container, string path) = await GetPartsAsync(fullPath, false).ConfigureAwait(false);
if (StoragePath.IsRootPath(path))
{
//deleting the entire container / filesystem
await container.DeleteIfExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
else
{
BlockBlobClient blob = string.IsNullOrEmpty(path)
? null
: container.GetBlockBlobClient(StoragePath.Normalize(path));
if (blob != null)
{
try
{
await blob.DeleteAsync(
DeleteSnapshotsOption.IncludeSnapshots, cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "BlobNotFound")
{
//this might be a folder reference, just try it
await foreach (BlobItem recursedFile in
container.GetBlobsAsync(prefix: path, cancellationToken: cancellationToken).ConfigureAwait(false))
{
BlobClient client = container.GetBlobClient(recursedFile.Name);
await client.DeleteIfExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
}
}
}
}
internal static ResourceFile UploadFileToBatchContainer(string storageAcctName, string containerName, StorageSharedKeyCredential storageCreds, string filePath)
{
log.LogInformation($"Uploading file {filePath} to container [{containerName}]...");
string blobName = Path.GetFileName(filePath);
// BlobContainerClient container = new BlobContainerClient(new Uri($"https://{storageAcctName}.blob.core.windows.net storage/{containerName}"), storageCreds); //SvcClient.GetBlobContainerClient(containerName);
BlockBlobClient blobData = new BlockBlobClient(new Uri($"https://{storageAcctName}.blob.core.windows.net/{containerName}/{blobName}"), storageCreds); //container.GetBlockBlobClient(blobName);
using (var fs = new FileStream(filePath, FileMode.Open))
{
blobData.Upload(fs);
}
// Set the expiry time and permissions for the blob shared access signature. In this case, no start time is specified,
// so the shared access signature becomes valid immediately
var sasPermissions = new BlobSasBuilder(BlobSasPermissions.Read, new DateTimeOffset(DateTime.UtcNow, new TimeSpan(0, 0, 0)));
sasPermissions.BlobName = blobName;
sasPermissions.BlobContainerName = containerName;
sasPermissions.StartsOn = DateTime.UtcNow.AddHours(-1);
sasPermissions.ExpiresOn = DateTime.UtcNow.AddHours(3);
// Construct the SAS URL for blob
blobData.GenerateSasUri(sasPermissions);
string blobSasUri = blobData.GenerateSasUri(sasPermissions).ToString();
return(ResourceFile.FromUrl(blobSasUri, blobName, null));
}
public async Task WriteAsync(string fullPath, Stream dataStream,
bool append = false, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
if (dataStream == null)
{
throw new ArgumentNullException(nameof(dataStream));
}
(BlobContainerClient container, string path) = await GetPartsAsync(fullPath, true).ConfigureAwait(false);
BlockBlobClient client = container.GetBlockBlobClient(path);
try
{
await client.UploadAsync(
new StorageSourceStream(dataStream),
cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "OperationNotAllowedInCurrentState")
{
//happens when trying to write to a non-file object i.e. folder
}
}
public async Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
(BlobContainerClient container, string path) = await GetPartsAsync(fullPath, false).ConfigureAwait(false);
BlockBlobClient client = container.GetBlockBlobClient(path);
try
{
//current SDK fails to download 0-sized files
Response <BlobProperties> p = await client.GetPropertiesAsync().ConfigureAwait(false);
if (p.Value.ContentLength == 0)
{
return(new MemoryStream());
}
Response <BlobDownloadInfo> response = await client.DownloadAsync(cancellationToken).ConfigureAwait(false);
return(response.Value.Content);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "BlobNotFound")
{
return(null);
}
}
private async Task AnonymizeBlobsInJsonFormat(ActivityInputData inputData, BlobContainerClient inputContainer, BlobContainerClient outputContainer, string inputBlobPrefix, string outputBlobPrefix)
{
IEnumerable <BlobItem> blobsInJsonFormat = inputContainer.GetBlobs(BlobTraits.None, BlobStates.None, inputBlobPrefix, default).Where(blob => IsInputFileInJsonFormat(blob.Name));
FhirEnumerableReader <BlobItem> reader = new FhirEnumerableReader <BlobItem>(blobsInJsonFormat);
Func <BlobItem, Task <string> > anonymizeBlobFunc = async(blob) =>
{
string outputBlobName = GetOutputBlobName(blob.Name, inputBlobPrefix, outputBlobPrefix);
Console.WriteLine($"[{blob.Name}]：Processing... output to container '{outputContainer.Name}'");
var inputBlobClient = new BlobClient(inputData.SourceStorageConnectionString, inputContainer.Name, blob.Name, BlobClientOptions.Value);
var outputBlobClient = new BlockBlobClient(inputData.DestinationStorageConnectionString, outputContainer.Name, outputBlobName, BlobClientOptions.Value);
if (inputData.SkipExistedFile && await outputBlobClient.ExistsAsync().ConfigureAwait(false))
{
Console.WriteLine($"[{blob.Name}]：'{outputBlobName}' already exist. Skip");
return(string.Empty);
}
await outputBlobClient.DeleteIfExistsAsync().ConfigureAwait(false);
await AnonymizeSingleBlobInJsonFormatAsync(inputBlobClient, outputBlobClient, blob.Name, inputBlobPrefix).ConfigureAwait(false);
return(string.Empty);
};
FhirPartitionedExecutor <BlobItem, string> executor = new FhirPartitionedExecutor <BlobItem, string>(reader, null, anonymizeBlobFunc);
executor.PartitionCount = Environment.ProcessorCount * 2;
executor.BatchSize = 1;
await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
}
private async Task <ResourceAccessRuleSet> DownloadBlobAsync(string id, string eTag)
{
BlockBlobClient blob = this.Container.GetBlockBlobClient(id);
// Can't use DownloadContentAsync because of https://github.com/Azure/azure-sdk-for-net/issues/22598
try
{
Response <BlobDownloadStreamingResult> response = await blob.DownloadStreamingAsync(
conditions : string.IsNullOrEmpty(eTag)?null : new BlobRequestConditions {
IfNoneMatch = new ETag(eTag !)
})
.ConfigureAwait(false);
int status = response.GetRawResponse().Status;
if (status == 304)
{
// NOP - we are quite happy to ignore that, as the blob hasn't changed.
return(null);
}
// Note: it is technically possible to use System.Text.Json to work directly from
// the UTF-8 data, which is more efficient than decoding to a .NET UTF-16 string
// first. However, we have to do this for the time being because we are in the world of
// IJsonSerializerSettingsProvider, where all serialization options are managed in
// terms of JSON.NET.
using BlobDownloadStreamingResult blobDownloadStreamingResult = response.Value;
BinaryData data = await BinaryData.FromStreamAsync(blobDownloadStreamingResult.Content).ConfigureAwait(false);
string ruleSetJson = data.ToString();
ResourceAccessRuleSet ruleSet = JsonConvert.DeserializeObject <ResourceAccessRuleSet>(ruleSetJson, this.serializerSettings);
ruleSet.ETag = response.Value.Details.ETag.ToString("G");
return(ruleSet);
}
private static async Task CreateBlobAsync(Guid id, BlockBlobClient blockBlobClient, IProgress <long> progress,
CancellationToken cancellationToken = default)
{
await using var connection = new SqlConnection(ConnectionString);
await using var command = connection.CreateCommand();
command.CommandText = "SELECT [Index], Data FROM Blobs ORDER BY [Index] WHERE BlobId = @p0";
command.Parameters.Add(id);
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
await using var reader = await command
.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken).ConfigureAwait(false);
var base64BlockIds = new List <string>();
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var base64BlockId = Convert.ToBase64String(BitConverter.GetBytes(reader.GetInt32(0)));
await using var source = reader.GetStream(1);
await blockBlobClient.StageBlockAsync(base64BlockId, source, null, null, progress, cancellationToken)
.ConfigureAwait(false);
base64BlockIds.Add(base64BlockId);
}
await blockBlobClient.CommitBlockListAsync(base64BlockIds, cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
public static List <ExpandoObject> ListAllFilesAsync(string containerName)
{
try
{
BlobContainerClient containerClient = GetContainer(containerName);
Pageable <BlobItem> blobs = containerClient.GetBlobs();
dynamic responseObject = new ExpandoObject();
responseObject.fileDownloadUrl = string.Empty;
foreach (BlobItem blob in blobs)
{
BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blob.Name);
// Download the blob's contents and save it to a file
BlobDownloadInfo download = blobClient.Download();
bytefile = ReadToEnd(download.Content);
UploadToSharePoint("dmsqa", "DMSWebJobOutPut", blob.Name);
//Console.WriteLine(blob.Name + " is Uploaded on sharepoint online");
}
return(null);
}
catch (Exception ex)
{
}
return(null);
}
public async Task GivenABlobFile_WhenExecutorWithoutAnonymize_DataShouldBeSame(string connectionString, string containerName, string blobName)
{
string targetContainerName = Guid.NewGuid().ToString("N");
string targetBlobName = Guid.NewGuid().ToString("N");
BlobContainerClient containerClient = new BlobContainerClient(connectionString, targetContainerName);
await containerClient.CreateIfNotExistsAsync();
try
{
BlobClient sourceBlobClient = new BlobClient(connectionString, containerName, blobName, DataFactoryCustomActivity.BlobClientOptions.Value);
BlockBlobClient targetBlobClient = new BlockBlobClient(connectionString, targetContainerName, targetBlobName, DataFactoryCustomActivity.BlobClientOptions.Value);
using FhirBlobDataStream stream = new FhirBlobDataStream(sourceBlobClient);
using FhirStreamReader reader = new FhirStreamReader(stream);
FhirBlobConsumer consumer = new FhirBlobConsumer(targetBlobClient);
var executor = new FhirPartitionedExecutor <string, string>(reader, consumer, content => content);
await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
Assert.Equal(sourceBlobClient.GetProperties().Value.ContentLength, targetBlobClient.GetProperties().Value.ContentLength);
}
finally
{
await containerClient.DeleteIfExistsAsync().ConfigureAwait(false);
}
}
/// <inheritdoc />
public async Task DeleteInstanceMetadataIfExistsAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
BlockBlobClient blob = GetInstanceBlockBlob(versionedInstanceIdentifier);
await ExecuteAsync(() => blob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, conditions: null, cancellationToken));
}
/// <inheritdoc/>
public async Task DeleteTenantAsync(string tenantId)
{
string parentTenantId = TenantExtensions.GetRequiredParentId(tenantId);
(_, BlobContainerClient parentContainer) =
await this.GetContainerAndTenantForChildTenantsOfAsync(parentTenantId).ConfigureAwait(false);
(_, BlobContainerClient tenantContainer) =
await this.GetContainerAndTenantForChildTenantsOfAsync(tenantId).ConfigureAwait(false);
// Check it's not empty first.
AsyncPageable <BlobItem> pageable = tenantContainer.GetBlobsAsync(
prefix: LiveTenantsPrefix);
IAsyncEnumerable <Page <BlobItem> > pages = pageable.AsPages(pageSizeHint: 1);
await using IAsyncEnumerator <Page <BlobItem> > page = pages.GetAsyncEnumerator();
if (await page.MoveNextAsync() && page.Current.Values.Count > 0)
{
throw new ArgumentException($"Cannot delete tenant {tenantId} because it has children");
}
BlockBlobClient blob = GetLiveTenantBlockBlobReference(tenantId, parentContainer);
Response <BlobDownloadResult> response = await blob.DownloadContentAsync().ConfigureAwait(false);
using var blobContent = response.Value.Content.ToStream();
BlockBlobClient?deletedBlob = parentContainer.GetBlockBlobClient(DeletedTenantsPrefix + tenantId);
await deletedBlob.UploadAsync(blobContent).ConfigureAwait(false);
await blob.DeleteIfExistsAsync().ConfigureAwait(false);
await tenantContainer.DeleteAsync().ConfigureAwait(false);
}
public void UploadBlockBlobStream()
{
// Get the connection string to storage account
string connectionString = ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString;
// Set the container name
string containerName = "testcontainer";
// Set the complete block blob path and name. You can also use just the name if you want to upload to root
string blobName = "parentDirectory/childDirectory/testBlockBlob.txt";
// Content we want to upload. We can also serialze an object to JSON here
// var obj = new MyObject { ID = 1, Name = "BlockBlob", SomeDate = DateTime.UtcNow };
// var content = Newtonsoft.Json.JsonConvert.SerializeObject(obj);
var content = "This content will be written to a block blob in storage account";
// Convert our contet to byte data
var bytesData = Encoding.UTF8.GetBytes(content);
// We are using the Blobs.Specialized.BlockBlobClient to quickly create block blob object with minimal code
BlockBlobClient blob = new BlockBlobClient(connectionString, containerName, blobName);
// Using memory stream to upload data
using (var memoryStream = new MemoryStream(bytesData))
{
blob.Upload(memoryStream);
}
}
// </Snippet_ListBlobsAnonymously>
//-------------------------------------------------
// Reference a blob anonymously
//-------------------------------------------------
// <Snippet_DownloadBlobAnonymously>
public static void DownloadBlobAnonymously()
{
BlockBlobClient blob = new BlockBlobClient
(new Uri(@"https://storagesamples.blob.core.windows.net/sample-container/logfile.txt"));
blob.DownloadTo(@"C:\Temp\logfile.txt");
}
public async Task QueryAsync_Large()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(250 * Constants.MB);
await blockBlobClient.UploadAsync(stream);
string query = @"SELECT * from BlobStorage";
// Act
TestProgress progressReporter = new TestProgress();
BlobQueryOptions options = new BlobQueryOptions
{
ProgressHandler = progressReporter
};
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options);
stream.Seek(0, SeekOrigin.Begin);
using StreamReader expectedStreamReader = new StreamReader(stream);
string expected = await expectedStreamReader.ReadToEndAsync();
using StreamReader actualStreamReader = new StreamReader(response.Value.Content);
string actual = await actualStreamReader.ReadToEndAsync();
// Assert
// Check we got back the same content that we uploaded.
Assert.AreEqual(expected, actual);
}
public async Task <string> TryAcquireLeaseAsync(BlockBlobClient blob, CancellationToken cancellationToken)
{
try
{
BlobLease lease = await blob.GetBlobLeaseClient().AcquireAsync(LeasePeriod, cancellationToken: cancellationToken).ConfigureAwait(false);
return(lease.LeaseId);
}
catch (RequestFailedException exception)
{
if (exception.IsConflictLeaseAlreadyPresent())
{
return(null);
}
else if (exception.IsNotFoundBlobOrContainerNotFound())
{
// If someone deleted the receipt, there's no lease to acquire.
return(null);
}
else
{
throw;
}
}
}
public async Task QueryAsync_AccessConditionsFail()
{
var garbageLeaseId = GetGarbageLeaseId();
foreach (AccessConditionParameters parameters in GetAccessConditionsFail_Data(garbageLeaseId))
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
parameters.NoneMatch = await SetupBlobMatchCondition(blockBlobClient, parameters.NoneMatch);
BlobRequestConditions accessConditions = BuildAccessConditions(parameters);
BlobQueryOptions options = new BlobQueryOptions
{
Conditions = accessConditions
};
string query = @"SELECT * from BlobStorage";
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blockBlobClient.QueryAsync(
query,
options),
e => { });
}
}
public async Task MarkCompletedAsync(BlockBlobClient blob, string leaseId,
CancellationToken cancellationToken)
{
var metadata = (await blob.GetPropertiesAsync(cancellationToken: cancellationToken).ConfigureAwait(false)).Value.Metadata;
BlobReceipt.Complete.ToMetadata(metadata);
try
{
await blob.SetMetadataAsync(
metadata,
new BlobRequestConditions()
{
LeaseId = leaseId,
},
cancellationToken : cancellationToken
).ConfigureAwait(false);
}
catch (RequestFailedException exception)
{
if (exception.IsNotFoundBlobOrContainerNotFound())
{
// The user deleted the receipt or its container; nothing to mark complete at this point.
}
else if (exception.IsPreconditionFailedLeaseLost())
{
// The lease expired; don't try to mark complete at this point.
}
else
{
throw;
}
}
}
public async Task QueryAsync_IfTags_Failed()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
BlobQueryOptions blobQueryOptions = new BlobQueryOptions
{
Conditions = new BlobRequestConditions
{
TagConditions = "\"coolTag\" = 'true'"
}
};
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blockBlobClient.QueryAsync(
querySqlExpression: query,
options: blobQueryOptions),
e => Assert.AreEqual(BlobErrorCode.ConditionNotMet.ToString(), e.ErrorCode));
}
/// <inheritdoc />
public async Task <Uri> StoreFileAsync(
VersionedInstanceIdentifier versionedInstanceIdentifier,
Stream stream,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
EnsureArg.IsNotNull(stream, nameof(stream));
BlockBlobClient blob = GetInstanceBlockBlob(versionedInstanceIdentifier);
stream.Seek(0, SeekOrigin.Begin);
var blobUploadOptions = new BlobUploadOptions {
TransferOptions = _options.Upload
};
try
{
await blob.UploadAsync(
stream,
blobUploadOptions,
cancellationToken);
return(blob.Uri);
}
catch (Exception ex)
{
throw new DataStoreException(ex);
}
}
public async Task QueryAsync_ArrowConfigurationInput()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
BlobQueryOptions options = new BlobQueryOptions
{
InputTextConfiguration = new BlobQueryArrowOptions
{
Schema = new List <BlobQueryArrowField>()
{
new BlobQueryArrowField
{
Type = BlobQueryArrowFieldType.Decimal,
Name = "Name",
Precision = 4,
Scale = 2
}
}
}
};
// Act
await TestHelper.AssertExpectedExceptionAsync <ArgumentException>(
blockBlobClient.QueryAsync(
query,
options: options),
e => Assert.AreEqual($"{nameof(BlobQueryArrowOptions)} can only be used for output serialization.", e.Message));
}
public BlobTest(TOptions options) : base(options)
{
var blobName = $"Azure.Storage.Blobs.Perf.BlobTest-{Guid.NewGuid()}";
BlobClient = BlobContainerClient.GetBlobClient(blobName);
BlockBlobClient = BlobContainerClient.GetBlockBlobClient(blobName);
}
