public string AddLabeledData()
{
string TrainingDataUrl;
CloudStorageAccount StorageAccount = _Engine.StorageAccount;
CloudBlobClient BlobClient = StorageAccount.CreateCloudBlobClient();
string labeledDataStorageContainerName = _Engine.GetEnvironmentVariable("labeledDataStorageContainerName", _Log);
CloudBlobContainer LabeledDataContainer = BlobClient.GetContainerReference(labeledDataStorageContainerName);
foreach (IListBlobItem item in LabeledDataContainer.ListBlobs(null, false))
{
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob dataCloudBlockBlob = (CloudBlockBlob)item;
TrainingDataUrl = dataCloudBlockBlob.Uri.ToString();
string BindingHash = dataCloudBlockBlob.Properties.ContentMD5.ToString();
if (BindingHash == null)
{
//compute the file hash as this will be added to the meta data to allow for file version validation
string BlobMd5 = new DataBlob(dataCloudBlockBlob, _Engine, _Search, _Log).CalculateMD5Hash().ToString();
if (BlobMd5 == null)
{
_Log.LogInformation("\nWarning: Blob Hash calculation failed and will not be included in file information blob, continuing operation.");
}
else
{
dataCloudBlockBlob.Properties.ContentMD5 = BlobMd5;
}
}
//trim the 2 "equals" off the trailing end of the hash or the http send will fail either using the client or raw http calls.
BindingHash = BindingHash.Substring(0, BindingHash.Length - 2);
//Get the content from the bound JSON file and instanciate a JsonBlob class then retrieve the labels collection from the Json to add to the image.
JsonBlob boundJson = (JsonBlob)_Search.GetBlob("json", BindingHash);
//Note you cannot pull the URL from the JSON blob because it will have the original URL from the first container when the blob was added to ML Professoar
string labeledDataUrl = dataCloudBlockBlob.StorageUri.PrimaryUri.ToString();
string addLabeledDataParameters = $"?dataBlobUrl={labeledDataUrl}";
string trainingDataLabels = Uri.EscapeDataString(JsonConvert.SerializeObject(boundJson.Labels));
addLabeledDataParameters = $"{addLabeledDataParameters}&imageLabels={trainingDataLabels}";
//construct and call model URL then fetch response
// the model always sends the label set in the message body with the name LabelsJson. If your model needs other values in the URL then use
//{ {environment variable name}}.
// So the example load labels function in the sameple model package would look like this:
// https://branddetectionapp.azurewebsites.net/api/loadimagetags/?projectID={{ProjectID}}
// The orchestration engine appends the labels json file to the message body.
// http://localhost:7071/api/LoadImageTags/?projectID=8d9d12d1-5d5c-4893-b915-4b5b3201f78e&labelsJson={%22Labels%22:[%22Hemlock%22,%22Japanese%20Cherry%22]}
string labeledDataServiceEndpoint = _Engine.GetEnvironmentVariable("LabeledDataServiceEndpoint", _Log);
string addLabeledDataUrl = _Engine.ConstructModelRequestUrl(labeledDataServiceEndpoint, addLabeledDataParameters);
_Log.LogInformation($"\n Getting response from {addLabeledDataUrl}");
_Response = _Client.GetAsync(addLabeledDataUrl).Result;
_ResponseString = _Response.Content.ReadAsStringAsync().Result;
if (string.IsNullOrEmpty(_ResponseString))
{
throw (new MissingRequiredObject($"\nresponseString not generated from URL: {addLabeledDataUrl}"));
}
//the code below is for passing labels and conent as http content and not on the URL string.
//Format the Data Labels content
//HttpRequestMessage Request = new HttpRequestMessage(HttpMethod.Post, new Uri(AddLabeledDataUrl));
//HttpContent DataLabelsStringContent = new StringContent(trainingDataLabels, Encoding.UTF8, "application/x-www-form-urlencoded");
//MultipartFormDataContent LabeledDataContent = new MultipartFormDataContent();
//LabeledDataContent.Add(DataLabelsStringContent, "LabeledData");
//Format the data cotent
//*****TODO***** move to an async architecture
//*****TODO***** need to decide if there is value in sending the data as a binary stream in the post or if requireing the model data scienctist to accept URLs is sufficient. If accessing the data blob with a SAS url requires Azure classes then create a configuration to pass the data as a stream in the post. If there is then this should be a configurable option.
//MemoryStream dataBlobMemStream = new MemoryStream();
//dataBlob.DownloadToStream(dataBlobMemStream);
//HttpContent LabeledDataHttpContent = new StreamContent(dataBlobMemStream);
//LabeledDataContent.Add(LabeledDataContent, "LabeledData");
//Make the http call and get a response
//string AddLabelingTagsEndpoint = Engine.GetEnvironmentVariable("LabeledDataServiceEndpoint", log);
//if (string.IsNullOrEmpty(AddLabelingTagsEndpoint)) throw (new EnvironmentVariableNotSetException("LabeledDataServiceEndpoint environment variable not set"));
//string ResponseString = Helper.GetEvaluationResponseString(AddLabelingTagsEndpoint, LabeledDataContent, log);
//if (string.IsNullOrEmpty(ResponseString)) throw (new MissingRequiredObject("\nresponseString not generated from URL: " + AddLabelingTagsEndpoint));
_Log.LogInformation($"Completed call to add blob: {dataCloudBlockBlob.Name} with labels: {JsonConvert.SerializeObject(boundJson.Labels)} to model. The response string was: {_ResponseString}.");
}
}
return("Completed execution of AddLabeledData. See logs for success/fail details.");
}
public async Task <string> EvaluateData(string blobName)
{
try
{
double modelVerificationPercent = 0;
string modelValidationStorageContainerName = "";
string storageConnection = _Engine.GetEnvironmentVariable("AzureWebJobsStorage", _Log);
string pendingEvaluationStorageContainerName = _Engine.GetEnvironmentVariable("pendingEvaluationStorageContainerName", _Log);
string evaluatedDataStorageContainerName = _Engine.GetEnvironmentVariable("evaluatedDataStorageContainerName", _Log);
string jsonStorageContainerName = _Engine.GetEnvironmentVariable("jsonStorageContainerName", _Log);
string pendingSupervisionStorageContainerName = _Engine.GetEnvironmentVariable("pendingSupervisionStorageContainerName", _Log);
string confidenceJsonPath = _Engine.GetEnvironmentVariable("confidenceJSONPath", _Log);
double confidenceThreshold = Convert.ToDouble(_Engine.GetEnvironmentVariable("confidenceThreshold", _Log));
string modelType = _Engine.GetEnvironmentVariable("modelType", _Log);
if (modelType == "Trained")
{
string labeledDataStorageContainerName = _Engine.GetEnvironmentVariable("labeledDataStorageContainerName", _Log);
modelValidationStorageContainerName = _Engine.GetEnvironmentVariable("modelValidationStorageContainerName", _Log);
string pendingNewModelStorageContainerName = _Engine.GetEnvironmentVariable("pendingNewModelStorageContainerName", _Log);
modelVerificationPercent = Convert.ToDouble(_Engine.GetEnvironmentVariable("modelVerificationPercentage", _Log));
}
//------------------------This section retrieves the blob needing evaluation and calls the evaluation service for processing.-----------------------
// Create Reference to Azure Storage Account and the container for data that is pending evaluation by the model.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnection);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(pendingEvaluationStorageContainerName);
CloudBlockBlob rawDataBlob = container.GetBlockBlobReference(blobName);
DataBlob dataEvaluating = new DataBlob(rawDataBlob, _Engine, _Search, _Log);
if (dataEvaluating == null)
{
throw (new MissingRequiredObject("\nMissing dataEvaluating blob object."));
}
//compute the file hash as this will be added to the meta data to allow for file version validation
//the blob has to be "touched" or the properties will all be null
if (dataEvaluating.AzureBlob.Exists() != true)
{
throw new MissingRequiredObject($"\ndataEvaluating does not exist {dataEvaluating.AzureBlob.Name}");
}
;
string blobMd5 = _Engine.EnsureMd5(dataEvaluating);
//****Currently only working with public access set on blob folders
//Generate a URL with SAS token to submit to analyze image API
//string dataEvaluatingSas = GetBlobSharedAccessSignature(dataEvaluating);
string dataEvaluatingUrl = dataEvaluating.AzureBlob.Uri.ToString(); //+ dataEvaluatingSas;
//string dataEvaluatingUrl = "test";
//package the file contents to send as http request content
//MemoryStream DataEvaluatingContent = new MemoryStream();
//DataEvaluating.AzureBlob.DownloadToStreamAsync(DataEvaluatingContent);
//HttpContent DataEvaluatingStream = new StreamContent(DataEvaluatingContent);
var content = new MultipartFormDataContent();
//content.Add(DataEvaluatingStream, "Name");
//get environment variables used to construct the model request URL
string dataEvaluationServiceEndpoint = _Engine.GetEnvironmentVariable("DataEvaluationServiceEndpoint", _Log);
string evaluationDataParameterName = _Engine.GetEnvironmentVariable("evaluationDataParameterName", _Log);
string parameters = $"?{evaluationDataParameterName}={dataEvaluatingUrl}";
string evaluateDataUrl = _Engine.ConstructModelRequestUrl(dataEvaluationServiceEndpoint, parameters);
int retryLoops = 0;
string responseString = "";
do
{
//Make a request to the model service passing the file URL
responseString = _Engine.GetHttpResponseString(evaluateDataUrl, content);
//*****TODO***** "iteration" is a hard coded word that is specific to a model and needs to be a generic interface concept where the model must respond with an explicit success.
if (responseString.Contains("iteration"))
{
_Log.LogInformation($"\nEvaluation response: {responseString}.");
break;
}
retryLoops++;
await Task.Delay(1000);
if (retryLoops == 5)
{
_Log.LogInformation($"\nEvaluation of {evaluateDataUrl} failed 5 attempts with response: {responseString}");
}
} while (retryLoops < 5);
string strConfidence = null;
double confidence = 0;
JProperty responseProperty = new JProperty("Response", responseString);
if (responseString == "Model not trained.")
{
confidence = 0;
}
else
{
//deserialize response JSON, get confidence score and compare with confidence threshold
JObject analysisJson = JObject.Parse(responseString);
try
{
strConfidence = (string)analysisJson.SelectToken(confidenceJsonPath);
}
catch
{
throw (new MissingRequiredObject($"\nInvalid response string {responseString} generated from URL: {evaluateDataUrl}."));
}
if (strConfidence == null)
{
//*****TODO***** if this fails the file will sit in the pending evaluation state because the trigger will have processed the file but the file could not be processed. Need to figure out how to tell if a file failed processing so that we can reprocesses the file at a latter time.
throw (new MissingRequiredObject($"\nNo confidence value at {confidenceJsonPath} from environment variable ConfidenceJSONPath in response from model: {responseString}."));
}
confidence = (double)analysisJson.SelectToken(confidenceJsonPath);
}
//----------------------------This section collects information about the blob being analyzed and packages it in JSON that is then written to blob storage for later processing-----------------------------------
_Log.LogInformation("\nStarting construction of json blob.");
//create environment JSON object
JProperty environmentProperty = _Engine.GetEnvironmentJson(_Log);
JProperty evaluationPass = new JProperty("pass",
new JObject(
new JProperty("date", DateTime.Now),
environmentProperty,
new JProperty("request", evaluateDataUrl),
responseProperty
)
);
//Note: all json files get writted to the same container as they are all accessed either by discrete name or by azure search index either GUID or Hash.
CloudBlobContainer jsonContainer = blobClient.GetContainerReference(jsonStorageContainerName);
CloudBlockBlob rawJsonBlob = jsonContainer.GetBlockBlobReference(_Engine.GetEncodedHashFileName(dataEvaluating.AzureBlob.Properties.ContentMD5.ToString()));
// If the Json blob already exists then update the blob with latest pass iteration information
if (rawJsonBlob.Exists())
{
//Hydrate Json Blob
JsonBlob jsonBlob = new JsonBlob(blobMd5, _Engine, _Search, _Log);
JObject jsonBlobJObject = JObject.Parse(jsonBlob.AzureBlob.DownloadText());
// Add an evaluation pass to the Json blob
JArray evaluationHistory = (JArray)jsonBlobJObject.SelectToken("Passes");
AddEvaluationPass(evaluationPass, evaluationHistory);
// Upload blob changes to the cloud
await _Engine.UploadJsonBlob(jsonBlob.AzureBlob, jsonBlobJObject);
}
// If the Json blob does not exist create one and include the latest pass iteration information
else
{
JObject BlobAnalysis =
new JObject(
new JProperty("Id", Guid.NewGuid().ToString()),
new JProperty("IsDeleted", false),
new JProperty("Name", blobName),
new JProperty("Hash", blobMd5)
);
// Add state history information to Json blob
JArray stateChanges = new JArray();
AddStateChange(pendingEvaluationStorageContainerName, stateChanges);
JProperty stateHistory = new JProperty("StateHistory", stateChanges);
BlobAnalysis.Add(stateHistory);
// Add pass infromation to Json blob
JArray evaluations = new JArray();
AddEvaluationPass(evaluationPass, evaluations);
JProperty evaluationPasses = new JProperty("Passes", evaluations);
BlobAnalysis.Add(evaluationPasses);
CloudBlockBlob JsonCloudBlob = _Search.GetBlob(storageAccount, jsonStorageContainerName, _Engine.GetEncodedHashFileName(blobMd5));
JsonCloudBlob.Properties.ContentType = "application/json";
await _Engine.UploadJsonBlob(JsonCloudBlob, BlobAnalysis);
}
//--------------------------------This section processes the results of the analysis and transferes the blob to the container responsible for the next appropriate stage of processing.-------------------------------
//model successfully analyzed content
if (confidence >= confidenceThreshold)
{
EvaluationPassed(modelVerificationPercent, modelValidationStorageContainerName, evaluatedDataStorageContainerName, storageAccount, dataEvaluating);
}
//model was not sufficiently confident in its analysis
else
{
EvaluationFailed(blobName, pendingSupervisionStorageContainerName, storageAccount, dataEvaluating);
}
_Log.LogInformation($"C# Blob trigger function Processed blob\n Name:{blobName}");
}
catch (MissingRequiredObject e)
{
_Log.LogInformation($"\n{blobName} could not be analyzed because of a MissingRequiredObject with message: {e.Message}");
}
catch (Exception e)
{
_Log.LogInformation($"\n{blobName} could not be analyzed with message: {e.Message}");
}
return($"Evaluate data completed evaluating data blob: {blobName}");
}
