public static CloudBlockBlob GetCloudBlockBlob(this Model model)
{
var account = new Storage.Helper().GetStorageAccount();
var client = account.CreateCloudBlobClient();
return(new CloudBlockBlob(model.Search.Url, client));
}
public static void Run([TimerTrigger("0 */1 * * * *", RunOnStartup = false)]TimerInfo myTimer, ILogger log)
#endif
{
Engine engine = new Engine(log);
try
{
string responseString = "";
CloudStorageAccount storageAccount = engine.StorageAccount;
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
//*****TODO***** externalize labeled data container name.
CloudBlobContainer labeledDataContainer = blobClient.GetContainerReference("labeleddata");
Model model = new Model(log);
// with a container load training tags
if (labeledDataContainer.ListBlobs(null, false) != null)
{
//*****TODO***** Where should search be initialized? Azure search does not offer CLI calls to configure all of search so it needs to be initialized befor it can be used as a service. Look at putting it in engine. Recognize this is not the same thing as migrating search to a non-static mode and then newing it up.
//Search.InitializeSearch();
// Create Reference to Azure Storage Account
var storageHelper = new Storage.Helper();
var StorageAccount = storageHelper.GetStorageAccount();
var BlobClient = StorageAccount.CreateCloudBlobClient();
var LabeledDataContainer = BlobClient.GetContainerReference("labeleddata");
var Client = new HttpClient();
var Response = new HttpResponseMessage();
var ResponseString = "";
//Add full set set of labeled training data to the model
//*****TODO***** add logic to only add incremental labeled data to model
string addLabeledDataResult = model.AddLabeledData();
//Train model using latest labeled training data.
string trainingResultsString = model.Train();
string TrainingDataUrl;
foreach (var item in LabeledDataContainer.ListBlobs(null, false))
{
if (item.GetType() == typeof(CloudBlockBlob))
{
var dataCloudBlockBlob = (CloudBlockBlob)item;
TrainingDataUrl = dataCloudBlockBlob.Uri.ToString();
var BindingHash = dataCloudBlockBlob.Properties.ContentMD5.ToString();
if (BindingHash == null)
{
//compute the file hash as this will be added to the meta data to allow for file version validation
var BlobMd5 = dataCloudBlockBlob.ToString().CalculateMD5Hash();
if (BlobMd5 == null)
{
log.LogInformation("\nWarning: Blob Hash calculation failed and will not be included in file information blob, continuing operation.");
}
else
{
dataCloudBlockBlob.Properties.ContentMD5 = BlobMd5;
}
}
//trim the 2 "equals" off the trailing end of the hash or the http send will fail either using the client or raw http calls.
BindingHash = BindingHash.Substring(0, BindingHash.Length - 2);
//Get the content from the bound JSON file and instanciate a JsonBlob class then retrieve the labels collection from the Json to add to the image.
var json = storageHelper.DownloadBlobAsString(StorageAccount, "json", BindingHash);
var model = json.ToStorageModel();
var labels = Uri.EscapeDataString(JsonConvert.SerializeObject(model.Labels));
//construct and call model URL then fetch response
// the model always sends the label set in the message body with the name LabelsJson. If your model needs other values in the URL then use
//{ {environment variable name}}.
// So the example load labels function in the sameple model package would look like this:
// https://branddetectionapp.azurewebsites.net/api/loadimagetags/?projectID={{ProjectID}}
// The orchestration engine appends the labels json file to the message body.
// http://localhost:7071/api/LoadImageTags/?projectID=8d9d12d1-5d5c-4893-b915-4b5b3201f78e&labelsJson={%22Labels%22:[%22Hemlock%22,%22Japanese%20Cherry%22]}
var AddLabeledDataUrl = model.Search.Url;
AddLabeledDataUrl = ConstructModelRequestUrl(AddLabeledDataUrl, labels, log);
Response = Client.GetAsync(AddLabeledDataUrl).Result;
ResponseString = Response.Content.ReadAsStringAsync().Result;
if (string.IsNullOrEmpty(ResponseString)) throw new MissingRequiredObjectException($"\nresponseString not generated from URL: {AddLabeledDataUrl}");
//the code below is for passing labels and conent as http content and not on the URL string.
//Format the Data Labels content
//HttpRequestMessage Request = new HttpRequestMessage(HttpMethod.Post, new Uri(AddLabeledDataUrl));
//HttpContent DataLabelsStringContent = new StringContent(trainingDataLabels, Encoding.UTF8, "application/x-www-form-urlencoded");
//MultipartFormDataContent LabeledDataContent = new MultipartFormDataContent();
//LabeledDataContent.Add(DataLabelsStringContent, "LabeledData");
//Format the data cotent
//*****TODO***** move to an async architecture
//*****TODO***** need to decide if there is value in sending the data as a binary stream in the post or if requireing the model data scienctist to accept URLs is sufficient. If accessing the data blob with a SAS url requires Azure classes then create a configuration to pass the data as a stream in the post. If there is then this should be a configurable option.
//MemoryStream dataBlobMemStream = new MemoryStream();
//dataBlob.DownloadToStream(dataBlobMemStream);
//HttpContent LabeledDataHttpContent = new StreamContent(dataBlobMemStream);
//LabeledDataContent.Add(LabeledDataContent, "LabeledData");
//Make the http call and get a response
//string AddLabelingTagsEndpoint = Engine.GetEnvironmentVariable("LabeledDataServiceEndpoint", log);
//if (string.IsNullOrEmpty(AddLabelingTagsEndpoint)) throw (new EnvironmentVariableNotSetException("LabeledDataServiceEndpoint environment variable not set"));
//string ResponseString = Helper.GetEvaluationResponseString(AddLabelingTagsEndpoint, LabeledDataContent, log);
//if (string.IsNullOrEmpty(ResponseString)) throw (new MissingRequiredObject("\nresponseString not generated from URL: " + AddLabelingTagsEndpoint));
log.LogInformation($"Successfully added blob: {dataCloudBlockBlob.Name} with labels: {JsonConvert.SerializeObject(model.Labels)}");
}
}
//Invoke the train model web service call
var trainModelUrl = Engine.GetEnvironmentVariable("TrainModelServiceEndpoint", log);
if (string.IsNullOrEmpty(trainModelUrl)) throw new EnvironmentVariableNotSetException("TrainModelServiceEndpoint environment variable not set");
Client = new HttpClient();
Response = Client.GetAsync(trainModelUrl).Result;
ResponseString = Response.Content.ReadAsStringAsync().Result;
if (string.IsNullOrEmpty(ResponseString)) throw new MissingRequiredObjectException($"\nresponseString not generated from URL: {trainModelUrl}");
log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");
}
catch (Exception e)
{
log.LogInformation("\nError processing training timer: ", e.Message);
}
}
public static async Task RunAsync([BlobTrigger("pendingevaluation/{blobName}", Connection = "AzureWebJobsStorage")] Stream myBlob, string blobName, ILogger log)
{
try
{
// need to add/fix json storage so there is only one container and need to
var PendingEvaluationStorageContainerName = GetEnvironmentVariable("pendingEvaluationStorageContainerName", log);
var EvaluatedDataStorageContainerName = GetEnvironmentVariable("evaluatedDataStorageContainerName", log);
var JsonStorageContainerName = GetEnvironmentVariable("jsonStorageContainerName", log);
var PendingSupervisionStorageContainerName = GetEnvironmentVariable("pendingSupervisionStorageContainerName", log);
var LabeledDataStorageContainerName = GetEnvironmentVariable("labeledDataStorageContainerName", log);
var ModelValidationStorageContainerName = GetEnvironmentVariable("modelValidationStorageContainerName", log);
var PendingNewModelStorageContainerName = GetEnvironmentVariable("pendingNewModelStorageContainerName", log);
var StorageConnection = GetEnvironmentVariable("AzureWebJobsStorage", log);
var ConfidenceJsonPath = GetEnvironmentVariable("confidenceJSONPath", log);
var DataTagsBlobName = GetEnvironmentVariable("dataTagsBlobName", log);
var ConfidenceThreshold = Convert.ToDouble(GetEnvironmentVariable("confidenceThreshold", log));
var ModelVerificationPercent = Convert.ToDouble(GetEnvironmentVariable("modelVerificationPercentage", log));
//------------------------This section retrieves the blob needing evaluation and calls the evaluation service for processing.-----------------------
// Create Reference to Azure Storage Account
var StorageAccount = CloudStorageAccount.Parse(StorageConnection);
var BlobClient = StorageAccount.CreateCloudBlobClient();
var Container = BlobClient.GetContainerReference(PendingEvaluationStorageContainerName);
//Get a reference to a container, if the container does not exist create one then get the reference to the blob you want to evaluate."
var json = new Storage.Helper().DownloadBlobAsString(StorageAccount, JsonStorageContainerName, blobName);
var model = json.ToStorageModel();
if (model == null)
{
throw new MissingRequiredObjectException("\nMissing dataEvaluating blob object.");
}
//compute the file hash as this will be added to the meta data to allow for file version validation
var BlobMd5 = model.ToString().CalculateMD5Hash();
if (BlobMd5 == null)
{
log.LogInformation("\nWarning: Blob Hash calculation failed and will not be included in file information blob, continuing operation.");
}
else
{
model.Md5Hash = BlobMd5;
}
//****Currently only working with public access set on blob folders
//Generate a URL with SAS token to submit to analyze image API
//string dataEvaluatingSas = GetBlobSharedAccessSignature(dataEvaluating);
var DataEvaluatingUrl = model.Search.Url.ToString(); //+ dataEvaluatingSas;
//string dataEvaluatingUrl = "test";
//package the file contents to send as http request content
var DataEvaluatingContent = new MemoryStream();
await model.GetAzureBlob().DownloadToStreamAsync(DataEvaluatingContent);
HttpContent DataEvaluatingStream = new StreamContent(DataEvaluatingContent);
var content = new MultipartFormDataContent();
content.Add(DataEvaluatingStream, "name");
//Make a request to the model service passing the file URL
var ResponseString = Helper.GetEvaluationResponseString(DataEvaluatingUrl, content, log);
if (ResponseString == "")
{
throw new MissingRequiredObjectException("\nresponseString not generated from URL: " + DataEvaluatingUrl);
}
//deserialize response JSON, get confidence score and compare with confidence threshold
var AnalysisJson = JObject.Parse(ResponseString);
var StrConfidence = (string)AnalysisJson.SelectToken(ConfidenceJsonPath);
var Confidence = (double)AnalysisJson.SelectToken(ConfidenceJsonPath);
if (StrConfidence == null)
{
throw new MissingRequiredObjectException("\nNo confidence value at " + ConfidenceJsonPath + " from environment variable ConfidenceJSONPath.");
}
//--------------------------------This section processes the results of the analysis and transferes the blob to the container responsible for the next appropriate stage of processing.-------------------------------
//model successfully analyzed content
if (Confidence >= ConfidenceThreshold)
{
var EvaluatedData = Search.GetBlob(StorageAccount, EvaluatedDataStorageContainerName, blobName, log);
if (EvaluatedData == null)
{
throw new MissingRequiredObjectException("\nMissing evaluatedData " + blobName + " destination blob in container " + EvaluatedDataStorageContainerName);
}
CopyAzureBlobToAzureBlob(StorageAccount, model.GetAzureBlob(), EvaluatedData, log).Wait();
//pick a random number of successfully analyzed content blobs and submit them for supervision verification.
var Rnd = new Random();
if (Math.Round(Rnd.NextDouble(), 2) <= ModelVerificationPercent)
{
var ModelValidation = Search.GetBlob(StorageAccount, ModelValidationStorageContainerName, blobName, log);
if (ModelValidation == null)
{
log.LogInformation("\nWarning: Model validation skipped for " + blobName + " because of missing evaluatedData " + blobName + " destination blob in container " + ModelValidationStorageContainerName);
}
else
{
MoveAzureBlobToAzureBlob(StorageAccount, model.AzureBlob, ModelValidation, log).Wait();
}
}
await model.AzureBlob.DeleteIfExistsAsync();
}
//model was not sufficiently confident in its analysis
else
{
var PendingSupervision = Search.GetBlob(StorageAccount, PendingSupervisionStorageContainerName, blobName, log);
if (PendingSupervision == null)
{
throw new MissingRequiredObjectException("\nMissing pendingSupervision " + blobName + " destination blob in container " + PendingSupervisionStorageContainerName);
}
MoveAzureBlobToAzureBlob(StorageAccount, model.AzureBlob, PendingSupervision, log).Wait();
}
//----------------------------This section collects information about the blob being analyzied and packages it in JSON that is then written to blob storage for later processing-----------------------------------
var BlobAnalysis =
new JObject(
new JProperty("id", Guid.NewGuid().ToString()),
new JProperty("blobInfo",
new JObject(
new JProperty("name", blobName),
new JProperty("url", model.AzureBlob.Uri.ToString()),
new JProperty("modified", model.AzureBlob.Properties.LastModified.ToString()),
new JProperty("hash", BlobMd5)
)
)
);
//create environment JSON object
var BlobEnvironment = Engine.GetEnvironmentJson(log);
BlobAnalysis.Add(BlobEnvironment);
BlobAnalysis.Merge(AnalysisJson);
//Note: all json files get writted to the same container as they are all accessed either by discrete name or by azure search index either GUID or Hash.
var JsonBlob = Search.GetBlob(StorageAccount, JsonStorageContainerName, (string)BlobAnalysis.SelectToken("blobInfo.id") + ".json", log);
JsonBlob.Properties.ContentType = "application/json";
var SerializedJson = JsonConvert.SerializeObject(BlobAnalysis, Formatting.Indented, new JsonSerializerSettings {
});
Stream MemStream = new MemoryStream(Encoding.UTF8.GetBytes(SerializedJson));
if (MemStream.Length != 0)
{
await JsonBlob.UploadFromStreamAsync(MemStream);
}
else
{
throw new ZeroLengthFileException("\nencoded JSON memory stream is zero length and cannot be writted to blob storage");
}
log.LogInformation($"C# Blob trigger function Processed blob\n Name:{blobName} \n Size: {myBlob.Length} Bytes");
}
catch (MissingRequiredObjectException e)
{
log.LogInformation("\n" + blobName + " could not be analyzed with message: " + e.Message);
}
catch (Exception e)
{
log.LogInformation("\n" + blobName + " could not be analyzed with message: " + e.Message);
}
}
