private void ThrowIfNotJsonLines(AvailableBlobEvent metadata)
{
if (!Equal("application/x-jsonlines", metadata.ContentType))
{
throw new ArgumentException($"Blob content is {metadata.ContentType} which is not supported in this job.");
}
}
private async Task ProcessFile(string uniqueMessageId, AvailableBlobEvent metadata, FileInfo blobLocalStorage)
{
// Note: Preferably this is done in a separate process, after persisting the message and blob to an internal storage.
if (!Equal("testdata", metadata.BlobType)) // This is the only supported BlobType by this job.
{
return;
}
using (var fileStream = blobLocalStorage.OpenRead())
{
using (var uncompressedStream = UncompressIfNeeded(metadata, fileStream))
{
using (var streamReader = new StreamReader(uncompressedStream, Encoding.UTF8))
{
ThrowIfNotJsonLines(metadata);
var batchOutputFolder = LocalStorageUtility.CreateMessageOutputFolder(_config.OutputDirectory, uniqueMessageId);
string jsonLine;
var i = 0;
while ((jsonLine = await streamReader.ReadLineAsync()) != null)
{
var testData = JsonConvert.DeserializeObject <TestData>(jsonLine);
Console.WriteLine(
$" - Imported id={testData.Id}, name={testData.Name}, # of subitems {testData.Subitems?.Length}");
await LocalStorageUtility.WriteToFile(batchOutputFolder, i ++, jsonLine);
}
await LocalStorageUtility.WriteMetadataToFile(batchOutputFolder, metadata);
}
}
}
}
/// <summary>
/// Process the blob that has arrived.
/// Ideally this would just dump the file to some persistent queue, and have a separate job process it.
/// By doing that, you would quickly receieve all available data and then safely and efficiently process it.
/// In this example, we just read each line, print some info about it and dump it to a file (in the same folder for each blob)
/// </summary>
private async Task ProcessFile(string uniqueMessageId, AvailableBlobEvent metadata, FileInfo blobLocalStorage)
{
if (!Equal("testdata", metadata.BlobType)) // This is the only supported BlobType by this job.
{
return;
}
using var fileStream = blobLocalStorage.OpenRead();
using var uncompressedStream = UncompressIfNeeded(metadata, fileStream);
using var streamReader = new StreamReader(uncompressedStream, Encoding.UTF8);
ThrowIfNotJsonLines(metadata);
var batchOutputFolder = LocalStorageUtility.CreateMessageOutputFolder(_config.OutputDirectory, uniqueMessageId);
string jsonLine;
var i = 0;
while ((jsonLine = await streamReader.ReadLineAsync()) != null)
{
// Deserialize the line according to the contract and do something to it (in this example, just print it and dump to a file)
var testData = JsonConvert.DeserializeObject <TestData>(jsonLine);
Console.WriteLine($" - Imported id={testData.Id}, name={testData.Name}, # of subitems {testData.Subitems?.Length}");
await LocalStorageUtility.WriteToFile(batchOutputFolder, i ++, jsonLine);
}
await LocalStorageUtility.WriteMetadataToFile(batchOutputFolder, metadata);
}
public static async Task WriteMetadataToFile(string folder, AvailableBlobEvent metadata)
{
if (folder != null && Directory.Exists(folder))
{
await File.WriteAllTextAsync(Path.Join(folder, "metadata.json"),
metadata.SerializeToJson());
}
}
public async Task DownloadBlobAsync(AvailableBlobEvent availableBlobEvent, FileSystemInfo blobLocalStorage)
{
var uriBuilder = new UriBuilder(new Uri(availableBlobEvent.Uri))
{
Query = _sasToken
};
var cloudBlockBlob = new CloudBlockBlob(uriBuilder.Uri);
await cloudBlockBlob.DownloadToFileAsync(blobLocalStorage.FullName, FileMode.Create);
}
public async Task DownloadBlobAsync(AvailableBlobEvent availableBlobEvent, FileSystemInfo blobLocalStorage, CancellationToken cancellationToken)
{
var sasUri = new UriBuilder(availableBlobEvent.Uri)
{
Query = FindToken(availableBlobEvent.Uri)
};
var client = new BlobClient(sasUri.Uri);
await client.DownloadToAsync(blobLocalStorage.FullName, cancellationToken);
}
private static Stream UncompressIfNeeded(AvailableBlobEvent metadata, Stream fileStream)
{
return(metadata.ContentEncoding?.ToLower() == "gzip" ? new GZipStream(fileStream, CompressionMode.Decompress) : fileStream);
}
private static void LogMessage(AvailableBlobEvent metadata)
{
Console.WriteLine(
$"Got message of type {metadata.BlobType} and with correlation-id {metadata.CorrelationId}, uri {metadata.Uri}. Content had format {metadata.ContentType}");
}
private static Stream UncompressIfNeeded(AvailableBlobEvent metadata, Stream fileStream)
{
return(Equals("gzip", metadata.ContentEncoding) ? new GZipStream(fileStream, CompressionMode.Decompress) : fileStream);
}
