public async Task TriggerProcessDocument([ServiceBusTrigger("%IncomingDocumentsQueue%", Connection = "IncomingDocumentServiceBusConnectionString")] Message message, [DurableClient] IDurableOrchestrationClient starter, ILogger log, ExecutionContext ec)
{
log.LogInformation($"{ec.FunctionName} function was triggered by receipt of service bus message {message.MessageId}");
var activity = message.ExtractActivity();
string payload = System.Text.Encoding.UTF8.GetString(message.Body);
var body = JObject.Parse(payload);
if (!CanProcessMessage(message.MessageId, body, log))
{
log.LogWarning($"Message {message.MessageId} was ignored!! Please see previous log items for reasons.");
return;
}
HorusSql.CheckAndCreateDatabaseIfNecessary(log);
string blobUrl = body["data"]["url"].ToString();
string contentType = body["data"]["contentType"].ToString();
var job = new DocumentProcessingJob {
StagingBlobUrl = blobUrl, ContentType = contentType
};
string orchestrationId = await starter.StartNewAsync("DocumentProcessor", null, job);
log.LogInformation($"{ec.FunctionName} processed message {message.MessageId}. Orchestration {orchestrationId} will process document: {blobUrl}");
}
private void CheckAndCreateDatabaseIfNecessary(ILogger log)
{
log.LogTrace($"Checking if processing database has been initialised");
HorusSql.CheckAndCreateDatabaseIfNecessary(log);
using (SqlConnection connection = new SqlConnection(scoresSQLConnectionString))
{
connection.Open();
SqlCommand command = connection.CreateCommand();
SqlDataReader reader;
command.Connection = connection;
log.LogTrace($"Checking if scores database has been initialised");
command.CommandText = "select name from sysobjects where name = 'ScoreSummary'";
using (reader = command.ExecuteReader())
{
if (reader.HasRows)
{
log.LogTrace("Table ScoreSummary exists no need to create database tables");
return;
}
}
log.LogInformation($"Creating tables in {connection.Database} database ..");
SqlTransaction transaction = connection.BeginTransaction("InitializeDatabase");
command.Transaction = transaction;
var commandStr = "If not exists (select name from sysobjects where name = 'GeneratedDocuments')" +
"CREATE TABLE[dbo].[GeneratedDocuments]([Id][int] IDENTITY(1, 1) NOT NULL, [Account] [nvarchar](50) NULL, [SingleName] [nvarchar](50) NULL, [AddressLine1] [nvarchar](50) NULL, [AddressLine2] [nvarchar](50) NULL, " +
"[PostalCode] [nvarchar](50) NULL, [City] [nvarchar](50) NULL, [Notes] [nvarchar](50) NULL, [DocumentNumber] [nvarchar](50) NOT NULL, [FileName] [nvarchar](50) NULL, [DocumentFormat] [nvarchar](50) NULL, " +
"[DocumentDate] [datetime2](7) NULL, [PreTaxTotalValue] [decimal](19, 5) NULL, [TaxTotalValue] [decimal](19, 5) NULL, [ShippingTotalValue] [decimal](19, 5) NULL, [GrandTotalValue] [decimal](19, 5) NULL, [LineNumber] [nvarchar](5) NOT NULL, " +
"[Title] [nvarchar](50) NULL, [Author] [nvarchar](50) NULL, [Isbn] [nvarchar](50) NULL, [Quantity] [decimal](19, 5) NULL, [Discount] [decimal](19, 5) NULL, [Price] [decimal](19, 5) NULL, [Taxable] [bit] NOT NULL, " +
"[GoodsValue] [decimal](19, 5) NULL, [DiscountValue] [decimal](19, 5) NULL, [DiscountedGoodsValue] [decimal](19, 5) NULL, [TaxableValue] [decimal](19, 5) NULL "+
"PRIMARY KEY CLUSTERED ([Id] ASC)WITH(STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF) ON[PRIMARY]) ON[PRIMARY]";
command.CommandText = commandStr;
command.ExecuteNonQuery();
log.LogTrace($"Table GeneratedDocuments was created.");
commandStr = "If not exists (select name from sysobjects where name = 'ScoreSummary')" +
"CREATE TABLE[dbo].[ScoreSummary]([Id][int] IDENTITY(1, 1) NOT NULL, " +
"[Team] [nvarchar](50) NOT NULL, [TotalScore][int] NOT NULL, [InspectionTime] [datetime2](7) NOT NULL " +
"PRIMARY KEY CLUSTERED ([Id] ASC)WITH(STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF) ON[PRIMARY]) ON[PRIMARY]";
command.CommandText = commandStr;
command.ExecuteNonQuery();
log.LogTrace($"Table ScoreSummary was created.");
commandStr = "If not exists (select name from sysobjects where name = 'ScoreDetail')" +
"CREATE TABLE[dbo].[ScoreDetail]([Id][int] IDENTITY(1, 1) NOT NULL, " +
"[Team][nvarchar](50) NOT NULL, [InspectionTime] [datetime2](7) NOT NULL, [Type] [nvarchar](50) NOT NULL, [Notes] [nvarchar] (max)NULL, [Score] [int] NOT NULL, [Status] [nvarchar](15) NOT NULL " +
"PRIMARY KEY CLUSTERED ([Id] ASC)WITH(STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF) ON[PRIMARY]) ON[PRIMARY]";
command.CommandText = commandStr;
command.ExecuteNonQuery();
log.LogTrace($"Table ScoreDetail was created.");
transaction.Commit();
}
}
public async Task TriggerTrainModel([ServiceBusTrigger("%TrainingQueue%", Connection = "IncomingDocumentServiceBusConnectionString")] Message message, [DurableClient] IDurableOrchestrationClient starter, ILogger log, ExecutionContext ec)
{
log.LogInformation($"{ec.FunctionName} function was triggered by receipt of service bus message {message.MessageId}");
string payload = System.Text.Encoding.UTF8.GetString(message.Body);
var trm = JsonConvert.DeserializeObject <TrainingRequestMessage>(payload);
HorusSql.CheckAndCreateDatabaseIfNecessary(log);
var job = new ModelTrainingJob {
BlobFolderName = trm.BlobFolderName,
BlobSasUrl = trm.BlobSasUrl,
DocumentFormat = trm.DocumentFormat,
IncludeSubFolders = "false",
UseLabelFile = "true"
};
string orchestrationId = await starter.StartNewAsync("ModelTrainer", null, job);
log.LogInformation($"{ec.FunctionName} processed message {message.MessageId}. Orchestration {orchestrationId}");
}
