public string EvaluateData(string blobName)
{
try
{
string PendingEvaluationStorageContainerName = _Engine.GetEnvironmentVariable("pendingEvaluationStorageContainerName", _Log);
string EvaluatedDataStorageContainerName = _Engine.GetEnvironmentVariable("evaluatedDataStorageContainerName", _Log);
string JsonStorageContainerName = _Engine.GetEnvironmentVariable("jsonStorageContainerName", _Log);
string PendingSupervisionStorageContainerName = _Engine.GetEnvironmentVariable("pendingSupervisionStorageContainerName", _Log);
string LabeledDataStorageContainerName = _Engine.GetEnvironmentVariable("labeledDataStorageContainerName", _Log);
string ModelValidationStorageContainerName = _Engine.GetEnvironmentVariable("modelValidationStorageContainerName", _Log);
string PendingNewModelStorageContainerName = _Engine.GetEnvironmentVariable("pendingNewModelStorageContainerName", _Log);
string StorageConnection = _Engine.GetEnvironmentVariable("AzureWebJobsStorage", _Log);
string ConfidenceJsonPath = _Engine.GetEnvironmentVariable("confidenceJSONPath", _Log);
string DataTagsBlobName = _Engine.GetEnvironmentVariable("dataTagsBlobName", _Log);
double ConfidenceThreshold = Convert.ToDouble(_Engine.GetEnvironmentVariable("confidenceThreshold", _Log));
double ModelVerificationPercent = Convert.ToDouble(_Engine.GetEnvironmentVariable("modelVerificationPercentage", _Log));
//------------------------This section retrieves the blob needing evaluation and calls the evaluation service for processing.-----------------------
// Create Reference to Azure Storage Account and the container for data that is pending evaluation by the model.
CloudStorageAccount StorageAccount = CloudStorageAccount.Parse(StorageConnection);
CloudBlobClient BlobClient = StorageAccount.CreateCloudBlobClient();
CloudBlobContainer Container = BlobClient.GetContainerReference(PendingEvaluationStorageContainerName);
//Get a reference to a container, if the container does not exist create one then get the reference to the blob you want to evaluate."
CloudBlockBlob RawDataBlob = _Search.GetBlob(StorageAccount, JsonStorageContainerName, blobName, _Log);
DataBlob DataEvaluating = new DataBlob(RawDataBlob.Properties.ContentMD5, _Log);
if (DataEvaluating == null)
{
throw (new MissingRequiredObject("\nMissing dataEvaluating blob object."));
}
//compute the file hash as this will be added to the meta data to allow for file version validation
string BlobMd5 = FrameworkBlob.CalculateMD5Hash(DataEvaluating.ToString());
if (BlobMd5 == null)
{
_Log.LogInformation("\nWarning: Blob Hash calculation failed and will not be included in file information blob, continuing operation.");
}
else
{
DataEvaluating.AzureBlob.Properties.ContentMD5 = BlobMd5;
}
//****Currently only working with public access set on blob folders
//Generate a URL with SAS token to submit to analyze image API
//string dataEvaluatingSas = GetBlobSharedAccessSignature(dataEvaluating);
string DataEvaluatingUrl = DataEvaluating.AzureBlob.Uri.ToString(); //+ dataEvaluatingSas;
//string dataEvaluatingUrl = "test";
//package the file contents to send as http request content
//MemoryStream DataEvaluatingContent = new MemoryStream();
//DataEvaluating.AzureBlob.DownloadToStreamAsync(DataEvaluatingContent);
//HttpContent DataEvaluatingStream = new StreamContent(DataEvaluatingContent);
var content = new MultipartFormDataContent();
//content.Add(DataEvaluatingStream, "name");
//Make a request to the model service passing the file URL
string ResponseString = Helper.GetEvaluationResponseString(DataEvaluatingUrl, content, _Log);
if (ResponseString == "")
{
throw (new MissingRequiredObject("\nresponseString not generated from URL: " + DataEvaluatingUrl));
}
//deserialize response JSON, get confidence score and compare with confidence threshold
JObject AnalysisJson = JObject.Parse(ResponseString);
string StrConfidence = (string)AnalysisJson.SelectToken(ConfidenceJsonPath);
double Confidence = (double)AnalysisJson.SelectToken(ConfidenceJsonPath);
if (StrConfidence == null)
{
throw (new MissingRequiredObject("\nNo confidence value at " + ConfidenceJsonPath + " from environment variable ConfidenceJSONPath."));
}
//--------------------------------This section processes the results of the analysis and transferes the blob to the container responsible for the next appropriate stage of processing.-------------------------------
//model successfully analyzed content
if (Confidence >= ConfidenceThreshold)
{
CloudBlockBlob EvaluatedData = _Search.GetBlob(StorageAccount, EvaluatedDataStorageContainerName, blobName, _Log);
if (EvaluatedData == null)
{
throw (new MissingRequiredObject("\nMissing evaluatedData " + blobName + " destination blob in container " + EvaluatedDataStorageContainerName));
}
_Engine.CopyAzureBlobToAzureBlob(StorageAccount, DataEvaluating.AzureBlob, EvaluatedData, _Log).Wait();
//pick a random number of successfully analyzed content blobs and submit them for supervision verification.
Random Rnd = new Random();
if (Math.Round(Rnd.NextDouble(), 2) <= ModelVerificationPercent)
{
CloudBlockBlob ModelValidation = _Search.GetBlob(StorageAccount, ModelValidationStorageContainerName, blobName, _Log);
if (ModelValidation == null)
{
_Log.LogInformation("\nWarning: Model validation skipped for " + blobName + " because of missing evaluatedData " + blobName + " destination blob in container " + ModelValidationStorageContainerName);
}
else
{
_Engine.MoveAzureBlobToAzureBlob(StorageAccount, DataEvaluating.AzureBlob, ModelValidation, _Log).Wait();
}
}
DataEvaluating.AzureBlob.DeleteIfExistsAsync();
}
//model was not sufficiently confident in its analysis
else
{
CloudBlockBlob PendingSupervision = _Search.GetBlob(StorageAccount, PendingSupervisionStorageContainerName, blobName, _Log);
if (PendingSupervision == null)
{
throw (new MissingRequiredObject("\nMissing pendingSupervision " + blobName + " destination blob in container " + PendingSupervisionStorageContainerName));
}
_Engine.MoveAzureBlobToAzureBlob(StorageAccount, DataEvaluating.AzureBlob, PendingSupervision, _Log).Wait();
}
//----------------------------This section collects information about the blob being analyzied and packages it in JSON that is then written to blob storage for later processing-----------------------------------
JObject BlobAnalysis =
new JObject(
new JProperty("id", Guid.NewGuid().ToString()),
new JProperty("blobInfo",
new JObject(
new JProperty("name", blobName),
new JProperty("url", DataEvaluating.AzureBlob.Uri.ToString()),
new JProperty("modified", DataEvaluating.AzureBlob.Properties.LastModified.ToString()),
new JProperty("hash", BlobMd5)
)
)
);
//create environment JSON object
JProperty BlobEnvironment = _Engine.GetEnvironmentJson(_Log);
BlobAnalysis.Add(BlobEnvironment);
BlobAnalysis.Merge(AnalysisJson);
//Note: all json files get writted to the same container as they are all accessed either by discrete name or by azure search index either GUID or Hash.
CloudBlockBlob JsonBlob = _Search.GetBlob(StorageAccount, JsonStorageContainerName, (string)BlobAnalysis.SelectToken("blobInfo.id") + ".json", _Log);
JsonBlob.Properties.ContentType = "application/json";
string SerializedJson = JsonConvert.SerializeObject(BlobAnalysis, Newtonsoft.Json.Formatting.Indented, new JsonSerializerSettings {
});
Stream MemStream = new MemoryStream(Encoding.UTF8.GetBytes(SerializedJson));
if (MemStream.Length != 0)
{
JsonBlob.UploadFromStreamAsync(MemStream);
}
else
{
throw (new ZeroLengthFileException("\nencoded JSON memory stream is zero length and cannot be writted to blob storage"));
}
_Log.LogInformation($"C# Blob trigger function Processed blob\n Name:{blobName}");
}
catch (MissingRequiredObject e)
{
_Log.LogInformation("\n" + blobName + " could not be analyzed with message: " + e.Message);
}
catch (Exception e)
{
_Log.LogInformation("\n" + blobName + " could not be analyzed with message: " + e.Message);
}
return($"Evaluate data completed evaluating data blob: {blobName}");
}
