public async Task QueryAsync_IfTags()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
Dictionary <string, string> tags = new Dictionary <string, string>
{
{ "coolTag", "true" }
};
await blockBlobClient.SetTagsAsync(tags);
BlobQueryOptions blobQueryOptions = new BlobQueryOptions
{
Conditions = new BlobRequestConditions
{
TagConditions = "\"coolTag\" = 'true'"
}
};
// Act
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
querySqlExpression : query,
options : blobQueryOptions);
using StreamReader streamReader = new StreamReader(response.Value.Content);
string s = await streamReader.ReadToEndAsync();
// Assert
Assert.AreEqual("400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n", s);
}
public async Task QueryAsync_ArrowConfigurationInput()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
BlobQueryOptions options = new BlobQueryOptions
{
InputTextConfiguration = new BlobQueryArrowOptions
{
Schema = new List <BlobQueryArrowField>()
{
new BlobQueryArrowField
{
Type = BlobQueryArrowFieldType.Decimal,
Name = "Name",
Precision = 4,
Scale = 2
}
}
}
};
// Act
await TestHelper.AssertExpectedExceptionAsync <ArgumentException>(
blockBlobClient.QueryAsync(
query,
options: options),
e => Assert.AreEqual($"{nameof(BlobQueryArrowOptions)} can only be used for output serialization.", e.Message));
}
public async Task QueryAsync_ParquetConfiguration()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
using FileStream stream = File.OpenRead(
$"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}Resources{Path.DirectorySeparatorChar}parquet.parquet");
await blockBlobClient.UploadAsync(stream);
// Act
string query = @"select * from blobstorage where id < 1;";
BlobQueryOptions options = new BlobQueryOptions
{
InputTextConfiguration = new BlobQueryParquetTextOptions()
};
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options : options);
// Assert
using StreamReader streamReader = new StreamReader(response.Value.Content);
string s = await streamReader.ReadToEndAsync();
// Assert
Assert.AreEqual("0,mdifjt55.ea3,mdifjt55.ea3\n", s);
}
public async Task QueryAsync_IfTags_Failed()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
BlobQueryOptions blobQueryOptions = new BlobQueryOptions
{
Conditions = new BlobRequestConditions
{
TagConditions = "\"coolTag\" = 'true'"
}
};
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blockBlobClient.QueryAsync(
querySqlExpression: query,
options: blobQueryOptions),
e => Assert.AreEqual(BlobErrorCode.ConditionNotMet.ToString(), e.ErrorCode));
}
public async Task QueryAsync_Progress()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
// Act
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
TestProgress progressReporter = new TestProgress();
BlobQueryOptions options = new BlobQueryOptions
{
ProgressHandler = progressReporter
};
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options);
using StreamReader streamReader = new StreamReader(response.Value.Content);
await streamReader.ReadToEndAsync();
Assert.AreEqual(2, progressReporter.List.Count);
Assert.AreEqual(Constants.KB, progressReporter.List[0]);
Assert.AreEqual(Constants.KB, progressReporter.List[1]);
}
public async Task QueryAsync_AccessConditionsFail()
{
var garbageLeaseId = GetGarbageLeaseId();
foreach (AccessConditionParameters parameters in GetAccessConditionsFail_Data(garbageLeaseId))
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
parameters.NoneMatch = await SetupBlobMatchCondition(blockBlobClient, parameters.NoneMatch);
BlobRequestConditions accessConditions = BuildAccessConditions(parameters);
BlobQueryOptions options = new BlobQueryOptions
{
Conditions = accessConditions
};
string query = @"SELECT * from BlobStorage";
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blockBlobClient.QueryAsync(
query,
options),
e => { });
}
}
public async Task QueryAsync_Large()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(250 * Constants.MB);
await blockBlobClient.UploadAsync(stream);
string query = @"SELECT * from BlobStorage";
// Act
TestProgress progressReporter = new TestProgress();
BlobQueryOptions options = new BlobQueryOptions
{
ProgressHandler = progressReporter
};
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options);
stream.Seek(0, SeekOrigin.Begin);
using StreamReader expectedStreamReader = new StreamReader(stream);
string expected = await expectedStreamReader.ReadToEndAsync();
using StreamReader actualStreamReader = new StreamReader(response.Value.Content);
string actual = await actualStreamReader.ReadToEndAsync();
// Assert
// Check we got back the same content that we uploaded.
Assert.AreEqual(expected, actual);
}
public async Task QueryAsync_ArrowConfiguration()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
// Act
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
BlobQueryOptions options = new BlobQueryOptions
{
OutputTextConfiguration = new BlobQueryArrowOptions
{
Schema = new List <BlobQueryArrowField>()
{
new BlobQueryArrowField
{
Type = BlobQueryArrowFieldType.Decimal,
Name = "Name",
Precision = 4,
Scale = 2
}
}
}
};
// Act
await blockBlobClient.QueryAsync(
query,
options : options);
}
public async Task QueryAsync_FatalError()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
string query = @"SELECT * from BlobStorage;";
BlobQueryJsonTextOptions jsonTextConfiguration = new BlobQueryJsonTextOptions
{
RecordSeparator = "\n"
};
BlobQueryOptions options = new BlobQueryOptions
{
InputTextConfiguration = jsonTextConfiguration
};
// Act - with no IBlobQueryErrorReceiver
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options);
using StreamReader streamReader = new StreamReader(response.Value.Content);
string s = await streamReader.ReadToEndAsync();
// Act - with IBlobQueryErrorReceiver
BlobQueryError expectedBlobQueryError = new BlobQueryError
{
IsFatal = true,
Name = "ParseError",
Description = "Unexpected token ',' at [byte: 3]. Expecting tokens '{', or '['.",
Position = 0
};
BlobQueryErrorHandler errorHandler = new BlobQueryErrorHandler(expectedBlobQueryError);
options = new BlobQueryOptions
{
InputTextConfiguration = jsonTextConfiguration,
};
options.ErrorHandler += errorHandler.Handle;
response = await blockBlobClient.QueryAsync(
query,
options);
using StreamReader streamReader2 = new StreamReader(response.Value.Content);
s = await streamReader2.ReadToEndAsync();
}
public async Task QueryAsync_ParquetOutputError()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
// Act
string query = @"select * from blobstorage where id < 1;";
BlobQueryOptions options = new BlobQueryOptions
{
OutputTextConfiguration = new BlobQueryParquetTextOptions()
};
await TestHelper.AssertExpectedExceptionAsync <ArgumentException>(
blockBlobClient.QueryAsync(
query, options),
e => Assert.AreEqual($"{nameof(BlobQueryParquetTextOptions)} can only be used for input serialization.", e.Message));
}
public async Task QueryAsync_QueryTextConfigurations()
{
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
// Act
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
BlobQueryCsvTextOptions csvTextConfiguration = new BlobQueryCsvTextOptions
{
ColumnSeparator = ",",
QuotationCharacter = '"',
EscapeCharacter = '\\',
RecordSeparator = "\n",
HasHeaders = false
};
BlobQueryJsonTextOptions jsonTextConfiguration = new BlobQueryJsonTextOptions
{
RecordSeparator = "\n"
};
BlobQueryOptions options = new BlobQueryOptions
{
InputTextConfiguration = csvTextConfiguration,
OutputTextConfiguration = jsonTextConfiguration
};
// Act
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options);
using StreamReader streamReader = new StreamReader(response.Value.Content);
string s = await streamReader.ReadToEndAsync();
// Assert
Assert.AreEqual("{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n", s);
}
public async Task QueryAsync_NonFatalError()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
byte[] data = Encoding.UTF8.GetBytes("100,pizza,300,400\n300,400,500,600\n");
using MemoryStream stream = new MemoryStream(data);
await blockBlobClient.UploadAsync(stream);
string query = @"SELECT _1 from BlobStorage WHERE _2 > 250;";
// Act - with no IBlobQueryErrorReceiver
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(query);
using StreamReader streamReader = new StreamReader(response.Value.Content);
string s = await streamReader.ReadToEndAsync();
// Act - with IBlobQueryErrorReceiver
BlobQueryError expectedBlobQueryError = new BlobQueryError
{
IsFatal = false,
Name = "InvalidTypeConversion",
Description = "Invalid type conversion.",
Position = 0
};
BlobQueryErrorHandler errorHandler = new BlobQueryErrorHandler(expectedBlobQueryError);
BlobQueryOptions options = new BlobQueryOptions();
options.ErrorHandler += errorHandler.Handle;
response = await blockBlobClient.QueryAsync(
query,
options);
using StreamReader streamReader2 = new StreamReader(response.Value.Content);
s = await streamReader2.ReadToEndAsync();
}
public async Task QueryAsync_ArrowConfiguration()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
// Act
string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
BlobQueryOptions options = new BlobQueryOptions
{
OutputTextConfiguration = new BlobQueryArrowOptions
{
Schema = new List <BlobQueryArrowField>()
{
new BlobQueryArrowField
{
Type = BlobQueryArrowFieldType.Decimal,
Name = "Name",
Precision = 4,
Scale = 2
}
}
}
};
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options : options);
MemoryStream memoryStream = new MemoryStream();
await response.Value.Content.CopyToAsync(memoryStream);
// Assert
Assert.AreEqual("/////4AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABAwAMAAAACAAIAAAABAAIAAAABAAAAAEAAAAUAAAAEAAUAAgABgAHAAwAAAAQABAAAAAAAAEHJAAAABQAAAAEAAAAAAAAAAgADAAEAAgACAAAAAQAAAACAAAABAAAAE5hbWUAAAAAAAAAAP////9wAAAAEAAAAAAACgAOAAYABQAIAAoAAAAAAwMAEAAAAAAACgAMAAAABAAIAAoAAAAwAAAABAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAP////+IAAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMDABgAAAAAAgAAAAAAAAAACgAYAAwABAAIAAoAAAA8AAAAEAAAACAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAABAAAAIAAAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAA", Convert.ToBase64String(memoryStream.ToArray()));
}
public async Task QueryAsync_AccessConditions()
{
var garbageLeaseId = GetGarbageLeaseId();
foreach (AccessConditionParameters parameters in AccessConditions_Data)
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
Stream stream = CreateDataStream(Constants.KB);
await blockBlobClient.UploadAsync(stream);
parameters.Match = await SetupBlobMatchCondition(blockBlobClient, parameters.Match);
parameters.LeaseId = await SetupBlobLeaseCondition(blockBlobClient, parameters.LeaseId, garbageLeaseId);
BlobRequestConditions accessConditions = BuildAccessConditions(
parameters: parameters,
lease: true);
BlobQueryOptions options = new BlobQueryOptions
{
Conditions = accessConditions
};
string query = @"SELECT * from BlobStorage";
// Act
Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
query,
options);
// Assert
Assert.IsNotNull(response.Value.Details.ETag);
}
}
private static async Task <List <OpportunityHistory> > DumpQueryCsv(BlockBlobClient blob, string query, bool headers, ILogger log, List <string> attributes)
{
try
{
List <OpportunityHistory> opportunities = new List <OpportunityHistory>();
var id = attributes.FindIndex(str => str == "Id");
var snapshotdate = attributes.FindIndex(str => str == "SinkModifiedOn");
var estimatedvalue = attributes.FindIndex(str => str == "estimatedvalue");
var name = attributes.FindIndex(str => str == "name");
var counter = 0;
var options = new BlobQueryOptions()
{
InputTextConfiguration = new BlobQueryCsvTextOptions()
{
HasHeaders = headers
},
OutputTextConfiguration = new BlobQueryCsvTextOptions()
{
HasHeaders = true
},
ProgressHandler = new Progress <long>((finishedBytes) => Console.Error.WriteLine($"Data read: {finishedBytes}"))
};
options.ErrorHandler += (BlobQueryError err) => {
};
// BlobDownloadInfo exposes a Stream that will make results available when received rather than blocking for the entire response.
using (var reader = new StreamReader((await blob.QueryAsync(
query,
options)).Value.Content))
{
using (var parser = new CsvReader(reader, new CsvConfiguration(CultureInfo.CurrentCulture)
{
HasHeaderRecord = true
}))
{
while (await parser.ReadAsync())
{
//get indexOf prop
log.LogInformation(String.Join(" ", parser.Context.Record));
DateTime parsedDT;
decimal parsedDec;
opportunities.Add(new OpportunityHistory
{
Id = new Guid(parser.Context.Record[0]),
EstimatedRevenue = decimal.TryParse(parser.Context.Record[2], out parsedDec)? parsedDec:0,
SnapshotDate = DateTime.TryParse(parser.Context.Record[1], out parsedDT) ? (DateTime?)parsedDT : null,
Topic = parser.Context.Record[3]
});
counter++;
log.LogInformation("Counter: " + counter.ToString());
}
}
}
return(opportunities);
}
catch (Exception ex)
{
log.LogError("Exception: " + ex.ToString());
return(null);
}
}
