public async Task <ModelTrainingJob> StartTraining([ActivityTrigger] ModelTrainingJob job, ILogger log, Microsoft.Azure.WebJobs.ExecutionContext ec)
{
var snip = $"Orchestration { job.OrchestrationId}: { ec.FunctionName} - ";
var orchestrationContainer = orchestrationBlobClient.GetContainerReference($"{job.OrchestrationId}");
job.OrchestrationContainerName = orchestrationContainer.Name;
await orchestrationContainer.CreateIfNotExistsAsync();
var uri = $"{recognizerServiceBaseUrl}{BaseConstants.FormRecognizerApiPath}";
JObject body = new JObject(
new JProperty("source", job.BlobSasUrl),
new JProperty("sourceFilter",
new JObject(
new JProperty("prefix", job.BlobFolderName),
new JProperty("includeSubFolders", job.IncludeSubFolders)
)
),
new JProperty("useLabelFile", job.UseLabelFile)
);
string json = body.ToString();
string getUrl = "";
log.LogTrace($"{snip} Training Request was prepared {job.BlobSasUrl}");
using (var content = new StringContent(json, System.Text.Encoding.UTF8, "application/json"))
{
client.DefaultRequestHeaders.Add(BaseConstants.OcpApimSubscriptionKey, recognizerApiKey);
HttpResponseMessage response = await client.PostAsync(uri, content);
if (response.IsSuccessStatusCode)
{
log.LogTrace($"{snip} Training Request was submitted. StatusCode {response.StatusCode}");
HttpHeaders headers = response.Headers;
if (headers.TryGetValues("location", out IEnumerable <string> values))
{
getUrl = values.First();
}
}
else
{
var test = await response.Content.ReadAsStringAsync();
throw new Exception($"That didnt work. Trying to submit model training request {test} request was {json} Response:{response.StatusCode.ToString()}");
}
}
job.RecognizerStatusUrl = getUrl;
log.LogInformation($"{snip} Training Request for Document Format {job.DocumentFormat} Completed successfully");
return(job);
}
public async Task <ModelTrainingJob> TrainingCompleted([ActivityTrigger] ModelTrainingJob job, ILogger log, Microsoft.Azure.WebJobs.ExecutionContext ec)
{
var snip = $"Orchestration { job.OrchestrationId}: { ec.FunctionName} -";
var mtr = HorusSql.UpdateModelTraining(job, log);
job.ModelVersion = mtr.ModelVersion.ToString();
var orchestrationContainer = orchestrationBlobClient.GetContainerReference(job.OrchestrationContainerName);
await orchestrationContainer.CreateIfNotExistsAsync();
var jobBlobName = $"{job.DocumentFormat}{BaseConstants.TrainingJobFileExtension}";
var jobBlob = orchestrationContainer.GetBlockBlobReference(jobBlobName);
await jobBlob.UploadTextAsync(JsonConvert.SerializeObject(job));
log.LogInformation($"{snip} - Completed successfully - Job blob {job.JobBlobName} was uploaded to container {job.OrchestrationContainerName}");
return(job);
}
public async Task TriggerTrainModel([ServiceBusTrigger("%TrainingQueue%", Connection = "IncomingDocumentServiceBusConnectionString")] Message message, [DurableClient] IDurableOrchestrationClient starter, ILogger log, ExecutionContext ec)
{
log.LogInformation($"{ec.FunctionName} function was triggered by receipt of service bus message {message.MessageId}");
string payload = System.Text.Encoding.UTF8.GetString(message.Body);
var trm = JsonConvert.DeserializeObject <TrainingRequestMessage>(payload);
HorusSql.CheckAndCreateDatabaseIfNecessary(log);
var job = new ModelTrainingJob {
BlobFolderName = trm.BlobFolderName,
BlobSasUrl = trm.BlobSasUrl,
DocumentFormat = trm.DocumentFormat,
IncludeSubFolders = "false",
UseLabelFile = "true"
};
string orchestrationId = await starter.StartNewAsync("ModelTrainer", null, job);
log.LogInformation($"{ec.FunctionName} processed message {message.MessageId}. Orchestration {orchestrationId}");
}
public async Task <ModelTrainingJob> TrainingErrorHandler([ActivityTrigger] ModelTrainingJob job, ILogger log, Microsoft.Azure.WebJobs.ExecutionContext ec)
{
try
{
var snip = $"Orchestration { job.OrchestrationId}: { ec.FunctionName} -";
var orchestrationContainer = orchestrationBlobClient.GetContainerReference(job.OrchestrationContainerName);
await orchestrationContainer.CreateIfNotExistsAsync();
var jobBlobName = $"{job.DocumentFormat}{BaseConstants.TrainingJobFileExtension}";
var jobBlob = orchestrationContainer.GetBlockBlobReference(jobBlobName);
await jobBlob.UploadTextAsync(JsonConvert.SerializeObject(job));
var exceptionBlobName = $"{job.DocumentFormat}{BaseConstants.ExceptionExtension}";
var exceptionBlob = orchestrationContainer.GetBlockBlobReference(exceptionBlobName);
await exceptionBlob.UploadTextAsync(JsonConvert.SerializeObject(job.Exception));
log.LogInformation($"{snip} - Exception Handled - Exception of Type {job.Exception.GetType()} added to blob {job.JobBlobName} was uploaded to container{job.OrchestrationContainerName}");
return(job);
}
catch (Exception ex)
{
throw new HorusTerminalException(ex);
}
}
public static ModelTrainingRecord UpdateModelTraining(ModelTrainingJob m, ILogger log)
{
using (SqlConnection connection = new SqlConnection(sqlConnectionString))
{
connection.Open();
SqlCommand command = connection.CreateCommand();
SqlTransaction transaction;
transaction = connection.BeginTransaction("TrainingRequestTransaction");
command.Connection = connection;
command.Transaction = transaction;
int currentVersion = 0;
int newVersion = 0;
try
{
command.CommandText = $"SELECT MAX(ModelVersion) AS Current_Version from ModelTraining WHERE DocumentFormat='{m.DocumentFormat}'";
SqlDataReader reader = command.ExecuteReader();
try
{
while (reader.Read())
{
if (reader["Current_Version"] != System.DBNull.Value)
{
currentVersion = Convert.ToInt32(reader["Current_Version"]);
}
else
{
currentVersion = 0;
}
}
}
finally
{
reader.Close();
}
newVersion = currentVersion + 1;
// Add the row
string insertClause = $"Insert into ModelTraining (DocumentFormat, ModelVersion, ModelId, CreatedDateTime, UpdatedDateTime, BlobSasUrl, BlobfolderName, IncludeSubFolders, UseLabelFile, AverageModelAccuracy, TrainingDocumentResults";
string valuesClause = $" VALUES ('{m.DocumentFormat}', '{newVersion}','{m.ModelId}', '{m.CreatedDateTime:yyyy-MM-dd HH:mm:ss.fff}', '{m.UpdatedDateTime:yyyy-MM-dd HH:mm:ss.fff}', '{m.BlobSasUrl}', '{m.BlobFolderName}','{m.IncludeSubFolders}', '{m.UseLabelFile}','{m.AverageModelAccuracy}','{m.TrainingDocumentResults}'";
insertClause += ") ";
valuesClause += ")";
command.CommandText = insertClause + valuesClause;
command.ExecuteNonQuery();
transaction.Commit();
}
catch (Exception e)
{
log.LogError($"Exception prevented writing training request for document format {m.DocumentFormat} model id={m.ModelId} to database (transaction was rolled back). Message is {e.Message}");
transaction.Rollback();
throw e;
}
ModelTrainingRecord mtr = new ModelTrainingRecord {
ModelId = m.ModelId, ModelVersion = newVersion, AverageModelAccuracy = m.AverageModelAccuracy, DocumentFormat = m.DocumentFormat, UpdatedDateTime = m.UpdatedDateTime
};
log.LogInformation($"Training request for document format {m.DocumentFormat}, version={newVersion}, model id={m.ModelId} was written to the database");
return(mtr);
}
}
public async Task <ModelTrainingJob> CheckTrainingStatus([ActivityTrigger] ModelTrainingJob job, ILogger log, Microsoft.Azure.WebJobs.ExecutionContext ec)
{
var snip = $"Orchestration { job.OrchestrationId}: { ec.FunctionName} -";
log.LogTrace($"{snip} Checking training request status");
string responseBody;
JObject jsonContent;
string jobStatus;
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", recognizerApiKey);
var response = await client.GetAsync(job.RecognizerStatusUrl);
if (response.IsSuccessStatusCode)
{
responseBody = await response.Content.ReadAsStringAsync();
jsonContent = JObject.Parse(responseBody);
if (jsonContent["modelInfo"]["status"] != null)
{
jobStatus = jsonContent["modelInfo"]["status"].ToString();
if (jobStatus.ToLower() == "ready")
{
job.ModelId = jsonContent["modelInfo"]["modelId"].ToString();
string dateAsString = DateTime.Now.ToString();
try
{
dateAsString = jsonContent["modelInfo"]["createdDateTime"].ToString();
}
catch { }
DateTime dateValue;
if (DateTime.TryParse(dateAsString, out dateValue))
{
job.CreatedDateTime = dateValue;
}
dateAsString = DateTime.Now.ToString();
try
{
dateAsString = jsonContent["modelInfo"]["lastUpdatedDateTime"].ToString();
}
catch { }
if (DateTime.TryParse(dateAsString, out dateValue))
{
job.UpdatedDateTime = dateValue;
}
string numberAsString = "0";
try
{
numberAsString = jsonContent["trainResult"]["averageModelAccuracy"].ToString();
}
catch { }
decimal numberValue = 0;
if (Decimal.TryParse(numberAsString, out numberValue))
{
job.AverageModelAccuracy = numberValue;
}
else
{
job.AverageModelAccuracy = 0;
}
job.TrainingDocumentResults = jsonContent["trainResult"]["trainingDocuments"].ToString();
}
if (jobStatus.ToLower() == "invalid")
{
throw new Exception($" Training failed. Status={jobStatus} The response body was {responseBody}");
}
job.LatestRecognizerStatus = jobStatus;
log.LogInformation($"{snip} Training Request Status is {jobStatus}");
return(job);
}
throw new Exception($" Training failed. Status=null The response body was {responseBody}");
}
throw new Exception($" Training failed. Http Status Code {response.StatusCode}");
}
