public static async Task Run([EventGridTrigger] EventGridEvent eventGridEvent, TraceWriter log)
{
log.Info($"Function trigger with data: {eventGridEvent.ToString()}");
var eventType = eventGridEvent.EventType;
var dataObject = eventGridEvent.Data as JObject;
var eventData = dataObject.ToObject <StorageBlobCreatedEventData>();
var imageUrl = eventData.Url;
log.Info($"Triggering with event {eventType} and url {imageUrl}");
var visualFeatureTypes = new List <VisualFeatureTypes>
{
VisualFeatureTypes.Adult,
VisualFeatureTypes.Categories,
VisualFeatureTypes.Color,
VisualFeatureTypes.Description
//VisualFeatureTypes.Faces,
//VisualFeatureTypes.ImageType,
//VisualFeatureTypes.Tags
};
var analyzeImage = await computerVisionApi.AnalyzeImageAsync(imageUrl, visualFeatureTypes);
var blobReference = await cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(imageUrl));
try
{
blobReference.Metadata.Add("IsAdultContent", analyzeImage.Adult.IsAdultContent.ToString());
blobReference.Metadata.Add("AdultScore", analyzeImage.Adult.AdultScore.ToString("P0").Replace(" ", ""));
blobReference.Metadata.Add("IsRacyContent", analyzeImage.Adult.IsRacyContent.ToString());
blobReference.Metadata.Add("RacyScore", analyzeImage.Adult.RacyScore.ToString("P0").Replace(" ", ""));
blobReference.Metadata.Add("Tags", string.Join(", ", analyzeImage.Description.Tags));
blobReference.Metadata.Add("Category", analyzeImage.Categories[0].Name);
blobReference.Metadata.Add("Description", analyzeImage.Description.Captions[0].Text);
blobReference.Metadata.Add("IsBWImg", analyzeImage.Color.IsBWImg.ToString());
await blobReference.SetMetadataAsync();
}
catch (Exception e)
{
log.Info($"Exception caught at: {DateTime.UtcNow}, Message: {e.Message}");
}
}
public static void Run([EventGridTrigger] EventGridEvent eventGridEvent,
[Blob("{data.fileUrl}", FileAccess.Read, Connection = "feedstore_STORAGE")] CloudBlockBlob inputBlob,
[Blob("parquet/{rand-guid}.parquet", FileAccess.Write, Connection = "feedstore_STORAGE")] CloudBlockBlob outputBlob,
TraceWriter log)
{
log.Info(eventGridEvent.ToString());
JObject dataObject = eventGridEvent.Data as JObject;
var eventData = dataObject.ToObject <EventHubCaptureFileCreatedEventData>();
log.Info(eventData.EventCount.ToString() + " events in record");
if (eventData.EventCount > 0)
{
log.Info("Got a real file");
Stream avroStream = inputBlob.OpenReadAsync().Result;
CloudBlobStream parquetStream = outputBlob.OpenWriteAsync().Result;
CreateParquetFile(avroStream, parquetStream);
parquetStream.Close();
}
}
public static async Task Run([EventGridTrigger] EventGridEvent eventGridEvent, ILogger log)
{
// Process Device Events from IoT Hub
// https://docs.microsoft.com/en-us/azure/event-grid/event-schema-iot-hub?tabs=event-grid-event-schema
_logger = log;
JObject deviceEventData = (JObject)JsonConvert.DeserializeObject(eventGridEvent.Data.ToString());
//log.LogInformation($"Event Type {eventGridEvent.EventType.ToString()}");
log.LogInformation(eventGridEvent.ToString());
// Sanity check
if (deviceEventData == null)
{
log.LogError("Invalid input : Event Data is NULL");
return;
}
if (_serviceClient == null)
{
// Create IoT Hub Service Client
// https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.devices.serviceclient?view=azure-dotnet
_serviceClient = ServiceClient.CreateFromConnectionString(_iotHubConnectionString);
}
if (_serviceClient == null)
{
log.LogError("Failed to create to Service Client");
return;
}
if (_registryManager == null)
{
// Create Registry Manager
// https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.devices.registrymanager?view=azure-dotnet
_registryManager = RegistryManager.CreateFromConnectionString(_iotHubConnectionString);
}
if (_registryManager == null)
{
log.LogError("Failed to create to Registry Manager");
return;
}
switch (eventGridEvent.EventType)
{
case IotHubDeviceConnected:
await ProcessDeviceConnected(deviceEventData, log);
break;
case IotHubDeviceDisconnected:
await ProcessDeviceDisconnected(deviceEventData, log);
break;
case IotHubDeviceCreated:
await ProcessDeviceCreated(deviceEventData, log);
break;
case IotHubDeviceDeleted:
await ProcessDeviceDeleted(deviceEventData, log);
break;
}
}
public static async Task Run(
[EventGridTrigger] EventGridEvent eventGridEvent,
ILogger log)
{
log.LogInformation(eventGridEvent.Subject);
string eventCallerName = "CafMeteorologyECTowerFcns.DataLakeTransientToRaw()";
string eventCallerVersion = "v0.1.7";
log.LogInformation(eventCallerVersion);
EtlEventService etlEventService = new EtlEventService(
eventCallerName,
eventCallerVersion,
"AzureFunction");
etlEventService.LogInformation(
eventCallerName,
eventCallerVersion,
eventGridEvent.ToString());
etlEventService.AddInput($"EventGridEvent.Subject: {eventGridEvent.Subject}");
etlEventService.LogInformation(
eventCallerName,
eventCallerVersion,
$"EventGridEvent.Data: {eventGridEvent.Data}");
// Authenticate the Function for access to blob containers
string objectId =
Environment.GetEnvironmentVariable("FUNCTION_OBJECT_ID");
ManagedIdentityCredential credential =
new ManagedIdentityCredential(objectId);
// Read parameters
string OUTPUT_CONTAINER =
Environment.GetEnvironmentVariable("OUTPUT_CONTAINER");
string PROJECT_ID =
Environment.GetEnvironmentVariable("PROJECT_ID");
string DATALAKE_ENDPOINT =
Environment.GetEnvironmentVariable("DATALAKE_ENDPOINT");
if (eventGridEvent.EventType != "Microsoft.Storage.BlobCreated")
{
string msg = "EventType not BlobCreated, aborting";
etlEventService.LogError(
eventCallerName,
eventCallerVersion,
msg);
throw new Exception(msg);
}
// Get info from the event
log.LogInformation("Parsing Event");
JsonDocument json =
JsonDocument.Parse(eventGridEvent.Data.ToString());
string apiCall = json.RootElement.GetProperty("api").GetString();
log.LogInformation($"api: {apiCall}");
if (!(apiCall == "FlushWithClose" |
apiCall == "PutBlob" |
apiCall == "PutBlockList" |
apiCall == "CopyBlob"))
{
string msg = "EventGridEvent api not completely committed, aborting";
log.LogInformation(msg);
return;
}
try
{
string inputBlobUri = json.RootElement
.GetProperty("url")
.GetString();
BlobUriBuilder inputBlobUriBuilder = new BlobUriBuilder(
new Uri(inputBlobUri));
// Get input blob contents
log.LogInformation("Creating blob container client");
var blobContainerClient =
new BlobContainerClient(
new Uri(
$"https://{inputBlobUriBuilder.Host}/{inputBlobUriBuilder.BlobContainerName}"),
credential);
var inputBlobClient = blobContainerClient.GetBlobClient(inputBlobUriBuilder.BlobName);
if (!inputBlobClient.Exists())
{
log.LogInformation("Blob does not exist, exiting");
return;
}
log.LogInformation("Found blob, downloading content");
BlobDownloadInfo download = await inputBlobClient.DownloadAsync();
string blobContent;
using (StreamReader reader = new StreamReader(download.Content))
blobContent = await reader.ReadToEndAsync();
string blobName =
Path.GetFileName(new Uri(inputBlobUri).AbsolutePath);
log.LogInformation($"Blob length: {blobContent.Length}");
if (blobContent.Length <= 0)
{
log.LogInformation("Blob is empty, exiting");
return;
}
// Get metadata from input blob
log.LogInformation("Parsing Blob into TOA5 metadata");
TOA5Extractor extractor = new TOA5Extractor(
blobName,
blobContent,
-8);
Metadata blobMetadata = extractor.GetMetadata();
string outputBlobDataset = GetOutputDatasetName(blobMetadata);
string outputBlobDirPath = GetOutputSubDirPath(blobName);
// Move blob
log.LogInformation("Moving blob");
AzureDataLakeService dataLakeService =
new AzureDataLakeService(
DATALAKE_ENDPOINT,
credential);
string outputBlobPath = $"{PROJECT_ID}/{outputBlobDataset}/{outputBlobDirPath}/{blobName}";
log.LogInformation(outputBlobPath);
string outputUri = await dataLakeService.MoveBlob(
inputBlobUriBuilder.BlobContainerName,
inputBlobUriBuilder.BlobName,
OUTPUT_CONTAINER,
outputBlobPath,
etlEventService);
log.LogInformation("Blob moved");
etlEventService.AddOutput(outputUri);
}
catch (XmlException xmlException)
{
log.LogError(xmlException.Message);
}
catch (Exception e)
{
etlEventService.LogError(
eventCallerName,
eventCallerVersion,
$"Exception occured: {e}");
throw new Exception("Error in function", e);
}
finally
{
// Write EtlEvent
EtlEventServiceConfig etlEventServiceConfig = new EtlEventServiceConfig()
{
Zone = OUTPUT_CONTAINER,
Project = PROJECT_ID,
Endpoint = new Uri(DATALAKE_ENDPOINT),
Credential = credential
};
log.LogInformation("Writing EtlEvent");
string etlEventUri = await etlEventService.WriteAsync(etlEventServiceConfig);
log.LogInformation($"Wrote EtlEvent to: {etlEventUri}");
}
}
public static void Run([EventGridTrigger] EventGridEvent eventGridEvent, TraceWriter log)
#endif
{
log.Info(eventGridEvent.ToString());
}
