public static async Task Run([EventGridTrigger] EventGridEvent eventGridEvent, TraceWriter log) { log.Info($"Function trigger with data: {eventGridEvent.ToString()}"); var eventType = eventGridEvent.EventType; var dataObject = eventGridEvent.Data as JObject; var eventData = dataObject.ToObject <StorageBlobCreatedEventData>(); var imageUrl = eventData.Url; log.Info($"Triggering with event {eventType} and url {imageUrl}"); var visualFeatureTypes = new List <VisualFeatureTypes> { VisualFeatureTypes.Adult, VisualFeatureTypes.Categories, VisualFeatureTypes.Color, VisualFeatureTypes.Description //VisualFeatureTypes.Faces, //VisualFeatureTypes.ImageType, //VisualFeatureTypes.Tags }; var analyzeImage = await computerVisionApi.AnalyzeImageAsync(imageUrl, visualFeatureTypes); var blobReference = await cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(imageUrl)); try { blobReference.Metadata.Add("IsAdultContent", analyzeImage.Adult.IsAdultContent.ToString()); blobReference.Metadata.Add("AdultScore", analyzeImage.Adult.AdultScore.ToString("P0").Replace(" ", "")); blobReference.Metadata.Add("IsRacyContent", analyzeImage.Adult.IsRacyContent.ToString()); blobReference.Metadata.Add("RacyScore", analyzeImage.Adult.RacyScore.ToString("P0").Replace(" ", "")); blobReference.Metadata.Add("Tags", string.Join(", ", analyzeImage.Description.Tags)); blobReference.Metadata.Add("Category", analyzeImage.Categories[0].Name); blobReference.Metadata.Add("Description", analyzeImage.Description.Captions[0].Text); blobReference.Metadata.Add("IsBWImg", analyzeImage.Color.IsBWImg.ToString()); await blobReference.SetMetadataAsync(); } catch (Exception e) { log.Info($"Exception caught at: {DateTime.UtcNow}, Message: {e.Message}"); } }
public static void Run([EventGridTrigger] EventGridEvent eventGridEvent, [Blob("{data.fileUrl}", FileAccess.Read, Connection = "feedstore_STORAGE")] CloudBlockBlob inputBlob, [Blob("parquet/{rand-guid}.parquet", FileAccess.Write, Connection = "feedstore_STORAGE")] CloudBlockBlob outputBlob, TraceWriter log) { log.Info(eventGridEvent.ToString()); JObject dataObject = eventGridEvent.Data as JObject; var eventData = dataObject.ToObject <EventHubCaptureFileCreatedEventData>(); log.Info(eventData.EventCount.ToString() + " events in record"); if (eventData.EventCount > 0) { log.Info("Got a real file"); Stream avroStream = inputBlob.OpenReadAsync().Result; CloudBlobStream parquetStream = outputBlob.OpenWriteAsync().Result; CreateParquetFile(avroStream, parquetStream); parquetStream.Close(); } }
public static async Task Run([EventGridTrigger] EventGridEvent eventGridEvent, ILogger log) { // Process Device Events from IoT Hub // https://docs.microsoft.com/en-us/azure/event-grid/event-schema-iot-hub?tabs=event-grid-event-schema _logger = log; JObject deviceEventData = (JObject)JsonConvert.DeserializeObject(eventGridEvent.Data.ToString()); //log.LogInformation($"Event Type {eventGridEvent.EventType.ToString()}"); log.LogInformation(eventGridEvent.ToString()); // Sanity check if (deviceEventData == null) { log.LogError("Invalid input : Event Data is NULL"); return; } if (_serviceClient == null) { // Create IoT Hub Service Client // https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.devices.serviceclient?view=azure-dotnet _serviceClient = ServiceClient.CreateFromConnectionString(_iotHubConnectionString); } if (_serviceClient == null) { log.LogError("Failed to create to Service Client"); return; } if (_registryManager == null) { // Create Registry Manager // https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.devices.registrymanager?view=azure-dotnet _registryManager = RegistryManager.CreateFromConnectionString(_iotHubConnectionString); } if (_registryManager == null) { log.LogError("Failed to create to Registry Manager"); return; } switch (eventGridEvent.EventType) { case IotHubDeviceConnected: await ProcessDeviceConnected(deviceEventData, log); break; case IotHubDeviceDisconnected: await ProcessDeviceDisconnected(deviceEventData, log); break; case IotHubDeviceCreated: await ProcessDeviceCreated(deviceEventData, log); break; case IotHubDeviceDeleted: await ProcessDeviceDeleted(deviceEventData, log); break; } }
public static async Task Run( [EventGridTrigger] EventGridEvent eventGridEvent, ILogger log) { log.LogInformation(eventGridEvent.Subject); string eventCallerName = "CafMeteorologyECTowerFcns.DataLakeTransientToRaw()"; string eventCallerVersion = "v0.1.7"; log.LogInformation(eventCallerVersion); EtlEventService etlEventService = new EtlEventService( eventCallerName, eventCallerVersion, "AzureFunction"); etlEventService.LogInformation( eventCallerName, eventCallerVersion, eventGridEvent.ToString()); etlEventService.AddInput($"EventGridEvent.Subject: {eventGridEvent.Subject}"); etlEventService.LogInformation( eventCallerName, eventCallerVersion, $"EventGridEvent.Data: {eventGridEvent.Data}"); // Authenticate the Function for access to blob containers string objectId = Environment.GetEnvironmentVariable("FUNCTION_OBJECT_ID"); ManagedIdentityCredential credential = new ManagedIdentityCredential(objectId); // Read parameters string OUTPUT_CONTAINER = Environment.GetEnvironmentVariable("OUTPUT_CONTAINER"); string PROJECT_ID = Environment.GetEnvironmentVariable("PROJECT_ID"); string DATALAKE_ENDPOINT = Environment.GetEnvironmentVariable("DATALAKE_ENDPOINT"); if (eventGridEvent.EventType != "Microsoft.Storage.BlobCreated") { string msg = "EventType not BlobCreated, aborting"; etlEventService.LogError( eventCallerName, eventCallerVersion, msg); throw new Exception(msg); } // Get info from the event log.LogInformation("Parsing Event"); JsonDocument json = JsonDocument.Parse(eventGridEvent.Data.ToString()); string apiCall = json.RootElement.GetProperty("api").GetString(); log.LogInformation($"api: {apiCall}"); if (!(apiCall == "FlushWithClose" | apiCall == "PutBlob" | apiCall == "PutBlockList" | apiCall == "CopyBlob")) { string msg = "EventGridEvent api not completely committed, aborting"; log.LogInformation(msg); return; } try { string inputBlobUri = json.RootElement .GetProperty("url") .GetString(); BlobUriBuilder inputBlobUriBuilder = new BlobUriBuilder( new Uri(inputBlobUri)); // Get input blob contents log.LogInformation("Creating blob container client"); var blobContainerClient = new BlobContainerClient( new Uri( $"https://{inputBlobUriBuilder.Host}/{inputBlobUriBuilder.BlobContainerName}"), credential); var inputBlobClient = blobContainerClient.GetBlobClient(inputBlobUriBuilder.BlobName); if (!inputBlobClient.Exists()) { log.LogInformation("Blob does not exist, exiting"); return; } log.LogInformation("Found blob, downloading content"); BlobDownloadInfo download = await inputBlobClient.DownloadAsync(); string blobContent; using (StreamReader reader = new StreamReader(download.Content)) blobContent = await reader.ReadToEndAsync(); string blobName = Path.GetFileName(new Uri(inputBlobUri).AbsolutePath); log.LogInformation($"Blob length: {blobContent.Length}"); if (blobContent.Length <= 0) { log.LogInformation("Blob is empty, exiting"); return; } // Get metadata from input blob log.LogInformation("Parsing Blob into TOA5 metadata"); TOA5Extractor extractor = new TOA5Extractor( blobName, blobContent, -8); Metadata blobMetadata = extractor.GetMetadata(); string outputBlobDataset = GetOutputDatasetName(blobMetadata); string outputBlobDirPath = GetOutputSubDirPath(blobName); // Move blob log.LogInformation("Moving blob"); AzureDataLakeService dataLakeService = new AzureDataLakeService( DATALAKE_ENDPOINT, credential); string outputBlobPath = $"{PROJECT_ID}/{outputBlobDataset}/{outputBlobDirPath}/{blobName}"; log.LogInformation(outputBlobPath); string outputUri = await dataLakeService.MoveBlob( inputBlobUriBuilder.BlobContainerName, inputBlobUriBuilder.BlobName, OUTPUT_CONTAINER, outputBlobPath, etlEventService); log.LogInformation("Blob moved"); etlEventService.AddOutput(outputUri); } catch (XmlException xmlException) { log.LogError(xmlException.Message); } catch (Exception e) { etlEventService.LogError( eventCallerName, eventCallerVersion, $"Exception occured: {e}"); throw new Exception("Error in function", e); } finally { // Write EtlEvent EtlEventServiceConfig etlEventServiceConfig = new EtlEventServiceConfig() { Zone = OUTPUT_CONTAINER, Project = PROJECT_ID, Endpoint = new Uri(DATALAKE_ENDPOINT), Credential = credential }; log.LogInformation("Writing EtlEvent"); string etlEventUri = await etlEventService.WriteAsync(etlEventServiceConfig); log.LogInformation($"Wrote EtlEvent to: {etlEventUri}"); } }
public static void Run([EventGridTrigger] EventGridEvent eventGridEvent, TraceWriter log) #endif { log.Info(eventGridEvent.ToString()); }