public async Task Run(
[EventGridTrigger] EventGridEvent eventGridEvent, ILogger logger)
{
logger.LogInformation($"Started the evaluation of the gift");
StorageBlobCreatedEventData storageBlobCreatedEventData =
((JObject)eventGridEvent.Data).ToObject <StorageBlobCreatedEventData>();
string blobUrl = storageBlobCreatedEventData.Url;
string fileName = Path.GetFileName(blobUrl);
logger.LogInformation($"Evaluating Gift {fileName}");
var isGiftProperlyWrapped = await _computerVisionHelper.IsPresentPerfectlyWrapped(blobUrl);
if (!isGiftProperlyWrapped)
{
logger.LogError($"Gift Not Correctly wrapped. SANTA NOT HAPPY!!!!!!!!!!!!!!");
}
else
{
logger.LogInformation($"Gift is correctly wrapped. SANTA HAPPY!!!!!!!");
}
}
public void Parse_ValidBlobCreatedEvent_ShouldSucceed()
{
// Arrange
const string topic = "/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx";
const string subject = "/blobServices/default/containers/event-container/blobs/finnishjpeg";
const string eventType = "Microsoft.Storage.BlobCreated";
const string id = "5647b67c-b01e-002d-6a47-bc01ac063360";
const string dataVersion = "1";
const string metadataVersion = "1";
const string api = "PutBlockList";
const string clientRequestId = "5c24a322-35c9-4b46-8ef5-245a81af7037";
const string requestId = "5647b67c-b01e-002d-6a47-bc01ac000000";
const string eTag = "0x8D58A5F0C6722F9";
const string contentType = "image/jpeg";
const int contentLength = 29342;
const string blobType = "BlockBlob";
const string url = "https://sample.blob.core.windows.net/event-container/finnish.jpeg";
const string sequencer = "00000000000000000000000000000094000000000017d503";
const string batchId = "69cd1576-e430-4aff-8153-570934a1f6e1";
string rawEvent = EventSamples.BlobCreateEvent;
var eventTime = DateTimeOffset.Parse("2018-03-15T10:25:17.7535274Z");
// Act
var eventGridMessage = EventGridParser.ParseFromData <StorageBlobCreatedEventData>(rawEvent);
// Assert
Assert.NotNull(eventGridMessage);
Assert.NotNull(eventGridMessage.Events);
Assert.Single(eventGridMessage.Events);
var eventGridEvent = eventGridMessage.Events.Single();
Assert.Equal(topic, eventGridEvent.Topic);
Assert.Equal(subject, eventGridEvent.Subject);
Assert.Equal(eventType, eventGridEvent.EventType);
Assert.Equal(eventTime, eventGridEvent.EventTime);
Assert.Equal(id, eventGridEvent.Id);
Assert.Equal(dataVersion, eventGridEvent.DataVersion);
Assert.Equal(metadataVersion, eventGridEvent.MetadataVersion);
Assert.NotNull(eventGridEvent.Data);
StorageBlobCreatedEventData eventPayload = eventGridEvent.GetPayload();
Assert.NotNull(eventPayload);
Assert.Equal(api, eventPayload.Api);
Assert.Equal(clientRequestId, eventPayload.ClientRequestId);
Assert.Equal(requestId, eventPayload.RequestId);
Assert.Equal(eTag, eventPayload.ETag);
Assert.Equal(contentType, eventPayload.ContentType);
Assert.Equal(contentLength, eventPayload.ContentLength);
Assert.Equal(blobType, eventPayload.BlobType);
Assert.Equal(url, eventPayload.Url);
Assert.Equal(sequencer, eventPayload.Sequencer);
Assert.NotNull(eventPayload.StorageDiagnostics);
var storageDiagnostics = Assert.IsType <JObject>(eventPayload.StorageDiagnostics);
Assert.Equal(batchId, storageDiagnostics["batchId"]);
}
public static async Task Run([EventGridTrigger] EventGridEvent blobCreatedEvent, [DurableClient] IDurableClient context, ILogger log)
{
log.LogInformation(blobCreatedEvent.Data.ToString());
StorageBlobCreatedEventData createdEvent = ((JObject)blobCreatedEvent.Data).ToObject <StorageBlobCreatedEventData>();
var batchId = ParseBatchId(createdEvent.Url);
var entityId = new EntityId("BatchTracker", batchId);
context.SignalEntityAsync(entityId, "Add").Wait();
}
public static async Task FnActionDelta(
[EventGridTrigger] EventGridEvent eventGridEvent,
ILogger log)
{
// pull the filename from the event
log.LogInformation($"Event received {eventGridEvent.Id}");
StorageBlobCreatedEventData data = (eventGridEvent.Data as JObject).ToObject <StorageBlobCreatedEventData>();
var fileName = string.Empty;
var uri = new Uri(data.Url);
fileName = System.IO.Path.GetFileName(uri.LocalPath);
log.LogInformation($"filename is {fileName}");
// check its the file we're interested in
if (!fileName.EndsWith(".delta"))
{
log.LogInformation($"Not interested in this event { fileName}");
return;
}
var storage = new StorageProvider();
var newFileName = $"{fileName}.completed";
var containerName = Environment.GetEnvironmentVariable("region");
// set to run as singleton so this is belt and brace
if (await storage.FileExistsBlobStorageAsync(containerName, newFileName))
{
log.LogInformation($"This delta received already and completed. No further action.");
return;
}
log.LogInformation($"delta {fileName} received");
//
// action delta
//
var newFile = new ServiceTagFile()
{
ChangeNumber = fileName
};
var stream = StringToStream(SerializeFile(newFile));
await storage.WriteStreamAsBlobToStorageAsync(stream, containerName, newFileName);
// enqueue reference to delta for function to write access restrictions
log.LogInformation($"C# function actioned {fileName} and wrote completed {newFileName}");
}
/// <summary>
///
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
private static bool IsEventProcessable(StorageBlobCreatedEventData data)
{
if (data != null)
{
if (data.Url != String.Empty && data.Url.ToUpper().Contains(THUMBNAIL_CONTAINER_NAME.ToUpper()))
{
return(false);
}
else
{
return(true);
}
}
return(false);
}
public void ThenOutputsMessage()
{
var id = Guid.NewGuid().ToString();
var timeStamp = DateTime.Parse("2018-01-01 00:00:00");
var data = new StorageBlobCreatedEventData();
var eventGridEvent = new EventGridEvent(id: id, subject: "", data: data, eventType: Events.BlobCreated, eventTime: timeStamp, dataVersion: "");
EventGridPump.Run(eventGridEvent, out var output, new StubLogger());
var expected = new Message {
MessageId = id
};
output.MessageId.Should().Be(expected.MessageId);
}
public static async Task Run(
[EventGridTrigger] Microsoft.Azure.WebJobs.Extensions.EventGrid.EventGridEvent imageEvent,
[Blob("{data.url}", FileAccess.Read)] Stream inBlob,
[Blob("{data.url}-license.txt", FileAccess.Write)] Stream outBlob,
TraceWriter log
)
{
log.Info($"EventGrid trigger fired: {imageEvent.EventType}");
imageData = imageEvent.Data.ToObject <StorageBlobCreatedEventData>();
// Analyze the plate data
var plate = await AnalyzeImageAsync(inBlob, log);
// Create output blob with results
new MemoryStream(Encoding.UTF8.GetBytes(plate)).CopyTo(outBlob);
}
/// <summary>
///
/// </summary>
/// <param name="createdEvent"></param>
/// <returns></returns>
private static byte[] GetImageDataFromEvent(StorageBlobCreatedEventData createdEvent, ILogger log)
{
HttpClient http = new HttpClient();
using var response = http.GetAsync(createdEvent.Url);
response.Wait();
using var byteArrayTask = response.Result.Content.ReadAsByteArrayAsync();
byteArrayTask.Wait();
if (byteArrayTask.IsCompletedSuccessfully)
{
return(byteArrayTask.Result);
}
else
{
log.LogError($"{response.Status} ");
}
return(null);
}
public static async Task RunAsync(
[EventGridTrigger] EventGridEvent eventGridEvent,
[Blob("{data.url}", FileAccess.Read)] Stream plate,
[Blob("{data.url}-result.txt", FileAccess.Write)] Stream outBlob,
TraceWriter log)
{
log.Info(eventGridEvent.Data.ToString());
// Cast event data to a Storage Blob Created Event
StorageBlobCreatedEventData data = ((JObject)eventGridEvent.Data).ToObject <StorageBlobCreatedEventData>();
log.Info($"About to analyze image: {data.Url}");
// Try to analyze the text using Cognitive Services
var result = await TryAnalyzeAsync(plate);
log.Info($"Found text: {result.gotText} with text: {result.plate}");
// If the text was parsed
if (result.gotText)
{
log.Info("Writing results.txt into storage");
new MemoryStream(Encoding.UTF8.GetBytes(result.plate)).CopyTo(outBlob);
}
// Else (unable to parse text)
else
{
log.Info("Sending event to event grid");
// Send an event to Azure Event Grid
await EmitEventAsync(new EventGridEvent()
{
Id = Guid.NewGuid().ToString(),
Subject = $"LicensePlate{eventGridEvent.Subject}",
EventType = "PlateNotRead",
EventTime = DateTime.Now,
Data = new { url = data.Url },
DataVersion = "1.0"
});
log.Info("Writing results.txt into storage");
new MemoryStream(Encoding.UTF8.GetBytes("NOT FOUND - event sent")).CopyTo(outBlob);
}
}
public FileVirusScanner(EventGridEvent blobCreatedEvent, Stream blobStream, ILogger logger)
{
this.blobCreatedEvent = blobCreatedEvent;
this.blobCreatedEventData = ((JObject)blobCreatedEvent.Data).ToObject <StorageBlobCreatedEventData>();
this.blobStream = blobStream;
this.logger = logger;
var(storageAccount, containerName, blobName) = GetBlobDetails(blobCreatedEvent, logger);
this.StorageAccount = storageAccount;
this.ContainerName = containerName;
this.BlobName = blobName;
logger.LogInformation($"Subject - {blobCreatedEvent.Subject}");
logger.LogInformation($"Topic - {blobCreatedEvent.Topic}");
logger.LogInformation($"Blob details {storageAccount} {containerName} {blobName}");
logger.LogInformation($"Url: {blobCreatedEventData.Url}");
logger.LogInformation($"Api operation: {blobCreatedEventData.Api}");
logger.LogInformation($"ContentType: {blobCreatedEventData.ContentType}");
logger.LogInformation($"Blob Stream Length: {blobStream.Length} bytes");
}
public async Task NotifyRegisteredOwner(
[EventGridTrigger] EventGridEvent eventGridEvent,
ILogger logger
)
{
StorageBlobCreatedEventData blobCreatedEventData =
((JObject)eventGridEvent.Data).ToObject <StorageBlobCreatedEventData>();
string blobName = BlobHelper.GetBlobName(blobCreatedEventData.Url);
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.NotifyVehicleOwnerStarted),
LoggingConstants.Template,
LoggingConstants.EventId.NotifyVehicleOwnerStarted.ToString(),
blobName,
LoggingConstants.ProcessingFunction.NotifyRegisteredOwner.ToString(),
LoggingConstants.ProcessStatus.Started.ToString(),
"Execution Started"
);
#endregion
try
{
var speedingTicket = await _dmvDbHandler
.GetSpeedingTicketInfoAsync(blobName)
.ConfigureAwait(false);
var registeredOwnerInfo = await _dmvDbHandler
.GetOwnerInformationAsync(
vehicleRegistrationNumber : speedingTicket.VehicleResgistrationNumber)
.ConfigureAwait(false);
var infractionImage = await _blobHandler
.DownloadBlobAsync(blobCreatedEventData.Url)
.ConfigureAwait(false);
var attachments = new List <OwnerNotificationMessageAttachment>
{
new OwnerNotificationMessageAttachment
{
Name = speedingTicket.TicketNumber,
Content = Convert.ToBase64String(infractionImage),
ContentType = "image/jpeg"
}
};
var notificationMessage =
CreateNotificationMessage
(
vehicleOwnerInfo: registeredOwnerInfo,
attachments: attachments,
ticketNumber: speedingTicket.TicketNumber,
infractionDate: speedingTicket.Date,
infractionDistrict: speedingTicket.District
);
await _ownerNotificationHandler
.NotifyOwnerAsync(notificationMessage)
.ConfigureAwait(false);
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.NotifyVehicleOwnerFinished),
LoggingConstants.Template,
LoggingConstants.EventId.NotifyVehicleOwnerFinished.ToString(),
blobName,
LoggingConstants.ProcessingFunction.NotifyRegisteredOwner.ToString(),
LoggingConstants.ProcessStatus.Finished.ToString(),
"Execution Finished"
);
#endregion
}
catch (Exception ex)
{
CustomEventData customEventData = new CustomEventData
{
ImageUrl = blobCreatedEventData.Url,
CustomEvent = CustomEvent.Exceptioned.ToString()
};
await _eventHandler.PublishEventToTopicAsync(customEventData)
.ConfigureAwait(false);
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.NotifyVehicleOwnerFinished),
LoggingConstants.Template,
LoggingConstants.EventId.NotifyVehicleOwnerFinished.ToString(),
blobName,
LoggingConstants.ProcessingFunction.NotifyRegisteredOwner.ToString(),
LoggingConstants.ProcessStatus.Failed.ToString(),
$"Execution Failed. Reason: {ex.Message}"
);
#endregion
}
}
public async Task ExtractRegistrationNumber(
[EventGridTrigger] EventGridEvent eventGridEvent,
ILogger logger
)
{
StorageBlobCreatedEventData blobCreatedEventData =
((JObject)eventGridEvent.Data).ToObject <StorageBlobCreatedEventData>();
string blobName = BlobHelper.GetBlobName(blobCreatedEventData.Url);
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.ExtractRegistrationNumberStarted),
LoggingConstants.Template,
LoggingConstants.EventId.ExtractRegistrationNumberStarted.ToString(),
blobName,
LoggingConstants.ProcessingFunction.ExtractRegistrationNumber.ToString(),
LoggingConstants.ProcessStatus.Started.ToString(),
"Execution Started"
);
#endregion
CustomEventData customEventData = new CustomEventData
{
ImageUrl = blobCreatedEventData.Url,
TicketNumber = blobName,
};
try
{
string registrationNumber = await _computerVisionHandler
.ExtractRegistrationNumberWithUrlAsync(blobCreatedEventData.Url)
.ConfigureAwait(false);
var metadata = await _blobHandler
.GetBlobMetadataAsync(blobCreatedEventData.Url);
customEventData.DistrictOfInfraction = metadata["districtofinfraction"];
customEventData.DateOfInfraction = eventGridEvent.EventTime.ToString("dd-MM-yyyy");
if (!string.IsNullOrWhiteSpace(registrationNumber))
{
customEventData.VehicleRegistrationNumber = registrationNumber;
customEventData.CustomEvent = CustomEvent.NumberExtractionCompleted.ToString();
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.ExtractRegistrationNumberFinished),
LoggingConstants.Template,
LoggingConstants.EventId.ExtractRegistrationNumberFinished.ToString(),
blobName,
LoggingConstants.ProcessingFunction.ExtractRegistrationNumber.ToString(),
LoggingConstants.ProcessStatus.Finished.ToString(),
"Execution Finished"
);
#endregion
}
else
{
customEventData.CustomEvent = CustomEvent.Exceptioned.ToString();
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.ExtractRegistrationNumberFinished),
LoggingConstants.Template,
LoggingConstants.EventId.ExtractRegistrationNumberFinished.ToString(),
blobName,
LoggingConstants.ProcessingFunction.ExtractRegistrationNumber.ToString(),
LoggingConstants.ProcessStatus.Failed.ToString(),
"Execution Failed. Reason: Failed to extract number plate from the image"
);
#endregion
}
}
catch (Exception ex)
{
customEventData.CustomEvent = CustomEvent.Exceptioned.ToString();
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.ExtractRegistrationNumberFinished),
LoggingConstants.Template,
LoggingConstants.EventId.ExtractRegistrationNumberFinished.ToString(),
blobName,
LoggingConstants.ProcessingFunction.ExtractRegistrationNumber.ToString(),
LoggingConstants.ProcessStatus.Failed.ToString(),
$"Execution Failed. Reason: {ex.Message}"
);
#endregion
}
await _eventHandler
.PublishEventToTopicAsync(customEventData)
.ConfigureAwait(false);
}
public static async Task Run([EventGridTrigger] EventGridEvent blobCreatedEvent,
[Blob("{data.url}", FileAccess.Read, Connection = "FBI-STORAGEACCT")] Stream myBlob,
ILogger log)
{
bool trbundles = true;
string strbundles = System.Environment.GetEnvironmentVariable("FBI-TRANSFORMBUNDLES");
if (!string.IsNullOrEmpty(strbundles) && (strbundles.ToLower().Equals("no") || strbundles.ToLower().Equals("false")))
{
trbundles = false;
}
StorageBlobCreatedEventData createdEvent = ((JObject)blobCreatedEvent.Data).ToObject <StorageBlobCreatedEventData>();
string name = createdEvent.Url.Substring(createdEvent.Url.LastIndexOf('/') + 1);
if (myBlob == null)
{
return;
}
log.LogInformation($"ImportFHIRBUndles: Processing file Name:{name} \n Size: {myBlob.Length}");
var cbclient = StorageUtils.GetCloudBlobClient(System.Environment.GetEnvironmentVariable("FBI-STORAGEACCT"));
StreamReader reader = new StreamReader(myBlob);
var text = await reader.ReadToEndAsync();
var trtext = (trbundles ? FHIRUtils.TransformBundle(text, log) : text);
var fhirbundle = await FHIRUtils.CallFHIRServer("", trtext, HttpMethod.Post, log);
var result = LoadErrorsDetected(trtext, fhirbundle, name, log);
//Bundle Post was Throttled we can retry
if (!fhirbundle.Success && fhirbundle.Status == System.Net.HttpStatusCode.TooManyRequests)
{
//Currently cannot use retry hints with EventGrid Trigger function bindings so we will throw and exception to enter eventgrid retry logic for FHIR Server throttling and do
//our own dead letter for internal errors or unrecoverable conditions
log.LogInformation($"ImportFHIRBUndles File Name:{name} is throttled...");
throw new Exception($"ImportFHIRBUndles: Transient Error File: {name}...Entering eventgrid retry process until success or ultimate failure to dead letter if configured.");
}
//No Errors move to processed container
if (fhirbundle.Success && ((JArray)result["errors"]).Count == 0 && ((JArray)result["throttled"]).Count == 0)
{
await StorageUtils.MoveTo(cbclient, "bundles", "bundlesprocessed", name, $"{name}.processed", log);
await StorageUtils.WriteStringToBlob(cbclient, "bundlesprocessed", $"{name}.processed.result", fhirbundle.Content, log);
log.LogInformation($"ImportFHIRBUndles Processed file Name:{name}");
}
//Handle Errors from FHIR Server of proxy
if (!fhirbundle.Success || ((JArray)result["errors"]).Count > 0)
{
await StorageUtils.MoveTo(cbclient, "bundles", "bundleserr", name, $"{name}.err", log);
await StorageUtils.WriteStringToBlob(cbclient, "bundleserr", $"{name}.err.response", fhirbundle.Content, log);
await StorageUtils.WriteStringToBlob(cbclient, "bundleserr", $"{name}.err.actionneeded", result.ToString(), log);
log.LogInformation($"ImportFHIRBUndles File Name:{name} had errors. Moved to deadletter bundleserr directory");
}
//Handle Throttled Requests inside of bundle so we will create a new bundle to retry
if (fhirbundle.Success && ((JArray)result["throttled"]).Count > 0)
{
var nb = NDJSONConverter.initBundle();
nb["entry"] = result["throttled"];
string fn = $"retry{Guid.NewGuid().ToString().Replace("-","")}.json";
await StorageUtils.MoveTo(cbclient, "bundles", "bundlesprocessed", name, $"{name}.processed", log);
await StorageUtils.WriteStringToBlob(cbclient, "bundlesprocessed", $"{name}.processed.result", fhirbundle.Content, log);
await StorageUtils.WriteStringToBlob(cbclient, "bundles", fn, nb.ToString(), log);
log.LogInformation($"ImportFHIRBUndles File Name:{name} had throttled resources in response bundle. Moved to processed..Created retry bunde {fn}");
}
}
public static async Task Run([EventGridTrigger] EventGridEvent blobCreatedEvent,
[Blob("{data.url}", FileAccess.Read, Connection = "FBI-STORAGEACCT")] Stream myBlob,
ILogger log)
{
int maxresourcesperbundle = 200;
var cbclient = StorageUtils.GetCloudBlobClient(System.Environment.GetEnvironmentVariable("FBI-STORAGEACCT"));
string mrbundlemax = System.Environment.GetEnvironmentVariable("FBI-MAXRESOURCESPERBUNDLE");
if (!string.IsNullOrEmpty(mrbundlemax))
{
if (!int.TryParse(mrbundlemax, out maxresourcesperbundle))
{
maxresourcesperbundle = 200;
}
}
StorageBlobCreatedEventData createdEvent = ((JObject)blobCreatedEvent.Data).ToObject <StorageBlobCreatedEventData>();
log.LogInformation($"NDJSONConverter: Processing blob at {createdEvent.Url}...");
string name = createdEvent.Url.Substring(createdEvent.Url.LastIndexOf('/') + 1);
JObject rv = initBundle();
int linecnt = 0;
int total = 0;
int bundlecnt = 0;
int errcnt = 0;
int fileno = 1;
//Stream myBlob = null;
StringBuilder errsb = new StringBuilder();
using (StreamReader reader = new StreamReader(myBlob))
{
string line;
while ((line = reader.ReadLine()) != null)
{
linecnt++;
JObject res = null;
try
{
res = JObject.Parse(line);
addResource(rv, res);
bundlecnt++;
total++;
}
catch (Exception e)
{
log.LogError($"NDJSONConverter: File {name} is in error or contains invalid JSON at line number {linecnt}:{e.Message}");
errsb.Append($"{line}\n");
errcnt++;
}
if (bundlecnt >= maxresourcesperbundle)
{
await StorageUtils.WriteStringToBlob(cbclient, "bundles", $"{name}-{fileno++}.json", rv.ToString(), log);
bundlecnt = 0;
rv = null;
rv = initBundle();
}
}
if (bundlecnt > 0)
{
await StorageUtils.WriteStringToBlob(cbclient, "bundles", $"{name}-{fileno++}.json", rv.ToString(), log);
}
await StorageUtils.MoveTo(cbclient, "ndjson", "ndjsonprocessed", name, $"{name}.processed", log);
if (errcnt > 0)
{
await StorageUtils.WriteStringToBlob(cbclient, "ndjsonerr", $"{name}.err", errsb.ToString(), log);
}
log.LogInformation($"NDJSONConverter: Processing file {name} completed with {total} resources created in {fileno-1} bundles with {errcnt} errors...");
}
}