public static string WriteJsonFromCsv(string path)
{
var utils = new CsvUtils();
using (var csvReadStream = File.OpenRead(path))
using (var reader = new StreamReader(csvReadStream))
{
List <ThermostatReading> records = utils.FromCsv(reader);
var components = Path.GetFileNameWithoutExtension(path).Split('_');
var log = ThermostatLog.ParseFileIdentifier(Path.GetFileNameWithoutExtension(path));
log.Readings = records;
string json = JsonConvert.SerializeObject(log, Formatting.Indented, new JsonSerializerSettings()
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
DateFormatHandling = DateFormatHandling.IsoDateFormat,
DateTimeZoneHandling = DateTimeZoneHandling.Utc
});
var outputPath = Path.ChangeExtension(path, "json");
File.WriteAllText(outputPath, json);
Console.WriteLine($"Wrote: {outputPath}");
return(outputPath);
}
}
private static async Task ProcessEvents(IEnumerable <EventGridEvent> eventGridEvents, IAsyncCollector <string> thermostatLogOutput, ILogger log)
{
foreach (EventGridEvent e in eventGridEvents)
{
if (e.EventType != EventTypes.StorageBlobCreatedEvent)
{
log.LogWarning($"Recieved event other than blob create, got: '{e.EventType}', skipping.");
continue;
}
if (e.Subject.StartsWith($"/blobServices/default/containers/{containerName}") == false)
{
log.LogInformation($"Received event for unmonitored container, skipping.");
continue;
}
var blobCreatedEvent = (StorageBlobCreatedEventData)e.Data;
var blobUri = new Uri(blobCreatedEvent.Url);
var filename = blobUri.Segments.Last();
if (filename.EndsWith(".csv") == false)
{
log.LogWarning($"Recieved event for non-csv file '{filename}', skipping.");
continue;
}
string accountName = e.Topic.Substring(e.Topic.LastIndexOf('/') + 1);
var blobClient = blobClients.GetOrAdd(accountName, (accName) =>
{
string connectionEnvironmentVariableName = $"STORAGECONNECTION_{accName}";
string connectionString = Environment.GetEnvironmentVariable(connectionEnvironmentVariableName)
?? throw new Exception($"Cannot find connection string environment variable '{connectionEnvironmentVariableName}'.");
CloudStorageAccount account = CloudStorageAccount.Parse(connectionString);
return(account.CreateCloudBlobClient());
});
var blob = await blobClient.GetBlobReferenceFromServerAsync(blobUri);
using (var stream = await blob.OpenReadAsync(AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions(), new OperationContext()))
using (var reader = new StreamReader(stream))
{
ThermostatLog thermostatLog = ThermostatLog.ParseFileIdentifier(Path.GetFileNameWithoutExtension(filename));
var csvUtils = new CsvUtils();
thermostatLog.Readings = csvUtils.FromCsv(reader);
string json = formatter.ToJson(thermostatLog);
await thermostatLogOutput.AddAsync(json);
await thermostatLogOutput.FlushAsync();
log.LogInformation($"Wrote {filename} to cosmos.");
}
}
}
public static async Task Run(
[QueueTrigger("%LoadQueueName%", Connection = "AzureWebJobsStorage")] LoadJob job,
ILogger log)
{
var blobClient = blobClients.GetOrAdd(job.Account, (accName) =>
{
string connectionEnvironmentVariableName = $"STORAGECONNECTION_{job.Account}";
string connectionString = Environment.GetEnvironmentVariable(connectionEnvironmentVariableName)
?? throw new Exception($"Cannot find connection string environment variable '{connectionEnvironmentVariableName}'.");
CloudStorageAccount account = CloudStorageAccount.Parse(connectionString);
return(account.CreateCloudBlobClient());
});
var container = blobClient.GetContainerReference(job.Container);
var inputBlock = new BufferBlock <ThermostatLog>();
var sendBlock = new ActionBlock <ThermostatLog>(async thermostatLog =>
{
var filename = $"{thermostatLog.GetFileIdentifier()}.csv";
var blob = container.GetBlockBlobReference(filename);
blob.Properties.ContentType = "text/csv";
using (var stream = await blob.OpenWriteAsync())
using (var writer = new StreamWriter(stream))
{
csvUtils.ToCsv(writer, thermostatLog.Readings);
}
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 16
});
inputBlock.LinkTo(sendBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
var random = new Random();
for (int i = 0; i < job.Size; i++)
{
inputBlock.Post(ThermostatLog.GenerateRandomLog(random));
}
inputBlock.Complete();
await sendBlock.Completion;
}
public static string WriteRandomCsv()
{
var random = new Random();
var log = ThermostatLog.GenerateRandomLog(random);
var path = Path.Combine(Directory.GetCurrentDirectory(), "temp", log.GetFileIdentifier() + ".csv");
var utils = new CsvUtils();
using (var stream = File.OpenWrite(path))
using (var writer = new StreamWriter(stream))
{
utils.ToCsv(writer, log.Readings);
}
Console.WriteLine($"Wrote: {path}");
return(path);
}
public string ToJson(ThermostatLog log)
{
return(JsonConvert.SerializeObject(log, settings));
}
