public async void CanReadBlobContents()
{
var sampleBlobName = Guid.NewGuid().ToString().Replace("-", String.Empty);
var blob = (CloudBlockBlob)null;
try
{
var account = CloudStorageAccount.Parse(Configuration.StorageConnectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(BLOB_TEST_CONTAINER_NAME);
await container.CreateIfNotExistsAsync();
// Create sample blob
blob = container.GetBlockBlobReference(sampleBlobName);
await blob.UploadTextAsync(TEST_BLOB_CONTENTS);
// Read blob using BlobReader
var blobReader = new AzureBlobReader(Configuration.StorageConnectionString, BLOB_TEST_CONTAINER_NAME);
var blobData = await blobReader.ReadFileDataAsync(sampleBlobName);
Assert.Equal(TEST_BLOB_CONTENTS, Encoding.UTF8.GetString(blobData));
}
finally
{
// Cleanup from test
if (blob != null)
{
await blob.DeleteAsync();
}
}
}
public async Task ReadFileAsync_ReadsFile_WhenBlobExists()
{
var testData = Encoding.ASCII.GetBytes("{\"id\":325}");
var azureBlobApi = _simulation
.AddApi()
.AddHandler("HEAD /devaccount1/books", new ApiResponse(500)) // retry policy
.AddHandler("HEAD /devaccount1/books", ApiResponse.Ok())
// see https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob
.AddHandler("GET /devaccount1/books/sample.json", _ => {
var headers = Headers.FromString("x-ms-blob-type: BlockBlob");
var body = Body.FromStream(new MemoryStream(testData));
return(ApiResponse.Ok(headers, body));
})
.Build();
await azureBlobApi.StartAsync();
var testSettings = new AzureBlobSettings
{
ConnectionString = $"DefaultEndpointsProtocol=http;AccountName=devaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:{azureBlobApi.Port}/devaccount1;",
ContainerName = "books"
};
var blobReader = new AzureBlobReader(testSettings);
var blobFile = await blobReader.ReadFileAsync("sample.json");
var apiCalls = azureBlobApi.ReceivedApiCalls.ToList();
Assert.Multiple(() => {
Assert.AreEqual(testData, blobFile.ToArray());
StringAssert.StartsWith("HEAD /devaccount1/books", apiCalls[0].Action);
Assert.AreEqual(500, apiCalls[0].Response.StatusCode);
StringAssert.StartsWith("HEAD /devaccount1/books", apiCalls[1].Action);
Assert.AreEqual(200, apiCalls[1].Response.StatusCode);
});
}
public async Task ReadFilePartAsync_ReadsFilePart_WhenBlobExists()
{
const string testString = " {\"id\":325}";
const int offset = 2;
const int length = 4;
var testData = Encoding.ASCII.GetBytes(testString);
var expectedData = new ReadOnlySpan <byte>(testData).Slice(offset, length).ToArray();
var azureBlobApi = _simulation
.AddApi()
.AddHandler("HEAD /devaccount1/books", ApiResponse.Ok())
// see https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob
// and https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-the-range-header-for-blob-service-operations
.AddHandler("GET /devaccount1/books/sample.json", ctx => {
var headers = Headers.FromString("x-ms-blob-type: BlockBlob");
var range = (string)ctx.Request.Headers["x-ms-range"]
?? ctx.Request.Headers["Range"];
var data = testData;
if (!string.IsNullOrEmpty(range))
{
var bytes = range.Split('=')[1].Split('-');
var start = int.Parse(bytes[0]);
if (!int.TryParse(bytes[1], out var end))
{
end = testData.Length - 1;
}
var count = end - start + 1;
data = new ReadOnlySpan <byte>(testData).Slice(start, count).ToArray();
headers["Content-Range"] = $"bytes {start}-{end}/{count}";
var body = Body.FromStream(new MemoryStream(data));
return(new ApiResponse(206, "Partial Content", headers: headers, body: body));
}
return(ApiResponse.Ok(headers, Body.FromStream(new MemoryStream(testData))));
})
.Build();
await azureBlobApi.StartAsync();
var testSettings = new AzureBlobSettings
{
ConnectionString = $"DefaultEndpointsProtocol=http;AccountName=devaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:{azureBlobApi.Port}/devaccount1;",
ContainerName = "books"
};
var blobReader = new AzureBlobReader(testSettings);
var blobData = await blobReader.ReadFilePartAsync("sample.json", offset, length);
Assert.AreEqual(expectedData, blobData);
}
public async Task ReadFileAsync_Throws_WhenBlobContainerDoesNotExist()
{
var azureBlobApi = _simulation
.AddApi()
.AddHandler("HEAD /devaccount1/magazines", ApiResponse.NotFound())
.Build();
await azureBlobApi.StartAsync();
var testSettings = new AzureBlobSettings
{
ConnectionString = $"DefaultEndpointsProtocol=http;AccountName=devaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:{azureBlobApi.Port}/devaccount1;",
ContainerName = "magazines"
};
var blobReader = new AzureBlobReader(testSettings);
var exception = Assert.ThrowsAsync <InvalidOperationException>(async() => await blobReader.ReadFileAsync("abc.json"));
Assert.AreEqual("Container 'magazines' does not exist.", exception.Message);
}
public async Task ExecutesFullIntegrationPass()
{
var jobStatusStore = new JobStatusStore(Configuration.StorageConnectionString, Configuration.JobStatusContainerName);
var blobReader = new AzureBlobReader(Configuration.StorageConnectionString, Configuration.FileUploadContainerName);
var extractor = new TextFileContentExtractor();
var searchIndex = new AzureSearchIndex(Configuration.SearchServiceName, Configuration.SearchAdminKey);
var docScorer = new TextDocumentScorer(searchIndex);
var workflow = new ParsingWorkflow(jobStatusStore, blobReader, extractor, searchIndex, docScorer);
var blobId = Guid.NewGuid().ToString().Replace("-", String.Empty);
var jobId = Guid.NewGuid().ToString().Replace("-", String.Empty);
var blobUri = String.Format("{0}/{1}", jobId, blobId);
var blobDetails = new BlobDetails();
blobDetails.ContainerName = Configuration.FileUploadContainerName;
blobDetails.FullBlobPath = blobUri;
blobDetails.DocumentId = blobId;
blobDetails.JobId = jobId;
var job = new JobStatus();
job.OriginalFileName = "not-real-file.txt";
job.IsComplete = false;
job.JobStartTime = DateTime.UtcNow;
job.JobId = jobId;
await jobStatusStore.UpdateStatusAsync(job);
await createSampleBlob(blobUri);
await workflow.ExecuteAsync(blobDetails);
job = await jobStatusStore.ReadStatusAsync(jobId);
var categoryCount = job.Categories.Length;
Assert.Equal(1, categoryCount);
Assert.Equal("Heavy Hitter", job.Categories[0]);
}
