internal void Remove() { blob.Delete(); }
public static void Run([BlobTrigger("telemetry-archive/willowtelemetry01/{name}", Connection = "StorageAccount")] Stream myBlob, string name, TraceWriter log) { DataSet ds = new DataSet( new SchemaElement <int>("ObjectID"), new SchemaElement <string>("Value"), new SchemaElement <string>("ClientID"), new SchemaElement <string>("TimeStamp") ); if (myBlob.Length == 508) { return; } // empty dataset //log.Info($"C# Blob trigger function Processed blob\n Name:{name} \n Size: {myBlob.Length} Bytes"); var serializer = AvroSerializer.CreateGeneric(Schema); var jsonSerializer = new JsonSerializer(); using (var reader = AvroContainer.CreateGenericReader(myBlob)) { using (var streamReader = new SequentialReader <object>(reader)) { var results = streamReader.Objects; foreach (var item in results) { var body = ((AvroRecord)item)[5]; // ["Body"] is the 5th element var json = System.Text.Encoding.Default.GetString((byte[])body); var array = (JArray)JsonConvert.DeserializeObject(json); var telemetry = jsonSerializer.Deserialize <Telemetry>(array[0].CreateReader()); ds.Add(telemetry.ObjectID, telemetry.Value, telemetry.ClientID, telemetry.TimeStamp.ToString()); } } } //log.Info($"{ds.RowCount} Records processed"); if (ds.RowCount == 0) { return; } storageAccount = CloudStorageAccount.Parse(storageAcct); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer blobContainer = blobClient.GetContainerReference(parquetContainerName); blobContainer.CreateIfNotExists(); name = name + ".parquet"; CloudBlob blob = blobContainer.GetBlobReference(blobName); CloudAppendBlob appendBlob = blobContainer.GetAppendBlobReference(name); if (appendBlob.Exists()) { appendBlob.Delete(DeleteSnapshotsOption.IncludeSnapshots); } log.Info($"===> {ds.RowCount} Records, {name}"); using (var ms = new MemoryStream()) { using (var writer = new ParquetWriter(ms)) { writer.Write(ds); } ms.Seek(0, SeekOrigin.Begin); appendBlob.UploadFromStream(ms); } }