private async ValueTask NlpBackgroundTask(
Uri requestedTextServiceRegulationIri,
CancellationToken stoppingToken)
{
if (stoppingToken.IsCancellationRequested)
{
_logger.LogInformation($"{Environment.NewLine}Stopped because of CancellationToken.{Environment.NewLine}");
return;
}
_logger.LogInformation($"{Environment.NewLine}Queued Background Task (starting): {requestedTextServiceRegulationIri}{Environment.NewLine}");
//
// Text Service
//
var regulationResource = new RegulationResource
{
Url = requestedTextServiceRegulationIri
};
_logger.LogInformation($"{Environment.NewLine}Asking Text Service for regulation {regulationResource.RegulationYear}-{regulationResource.RegulationMonth}-{regulationResource.RegulationDay}-{regulationResource.RegulationNumber}.{Environment.NewLine}");
var regulationFromTextService = await _textServiceApi.GetRegulation(
_textServiceSettings.ApiBaseUrl,
Convert.ToInt32(regulationResource.RegulationYear),
Convert.ToInt32(regulationResource.RegulationMonth),
Convert.ToInt32(regulationResource.RegulationDay),
Convert.ToInt32(regulationResource.RegulationNumber));
var chapterList = _textServiceHelper.SplitRegulationResponseIntoChapterList(regulationFromTextService);
_logger.LogInformation($"{Environment.NewLine}{chapterList.Count} chapters loaded successfully.{Environment.NewLine}");
//
// NLP Service
//
// Load NLP options
if (_nlpServiceSettings.RunAsTest)
{ // Just load test data
NlpResourceDictionary =
_nlpServiceHelper.GetNlpResourceTestDictionary();
}
else // Send request to NLP Service API
{
var nlpResourceListFromNlpService =
await _nlpServiceApi.GetNlpResourceList(
_nlpServiceSettings.ApiBaseUrl);
NlpResourceDictionary =
_nlpServiceHelper.MapNlpResources(
_nlpServiceSettings.ApiBaseUrl,
nlpResourceListFromNlpService);
}
foreach (var nlpResourceDictionaryItem in NlpResourceDictionary)
{
var selectedNlpResourceDictionary = nlpResourceDictionaryItem.Value;
_logger.LogInformation($"{Environment.NewLine}Asking NLP Service to identify information about {selectedNlpResourceDictionary.Title} ({selectedNlpResourceDictionary.Language}) in regulation {regulationResource.RegulationYear}-{regulationResource.RegulationMonth}-{regulationResource.RegulationDay}-{regulationResource.RegulationNumber}.{Environment.NewLine}");
for (var i = 0; i < chapterList.Count; i++)
{
var requestNumber = i + 1;
_logger.LogInformation($"{Environment.NewLine}Processing chapter {requestNumber} of {chapterList.Count}:{Environment.NewLine}");
JsonElement identifiedInformationInChapterTextData;
if (_nlpServiceSettings.RunAsTest)
{
identifiedInformationInChapterTextData =
_nlpServiceHelper.GetTestDataForIdentifyInformationInChapterTextData();
}
else // Send request to NLP Service API
{
identifiedInformationInChapterTextData =
await _nlpServiceApi.IdentifyInformationInChapterTextData(
chapterList[i],
selectedNlpResourceDictionary.Url);
}
var itemCountOfNlpServiceResponse =
_nlpServiceHelper.CountItemsInNlpServiceApiResponse(
identifiedInformationInChapterTextData);
if (itemCountOfNlpServiceResponse > 0)
{
_logger.LogInformation($"{Environment.NewLine}{itemCountOfNlpServiceResponse} detections in this chapter.{Environment.NewLine}");
IdentifiedInformationInChapterTextDataList.Add(identifiedInformationInChapterTextData);
}
else
{
_logger.LogInformation($"{Environment.NewLine}No detections in this chapter.{Environment.NewLine}");
}
}
}
//
// Transformer Service
//
if (_transformerServiceSettings.RunAsTest)
{
_logger.LogInformation($"{Environment.NewLine}RUN AS TEST -> Asking Transformer Service to transform information into knowledge.{Environment.NewLine}");
var transformerCounter = 0;
foreach (var identifiedInformationInChapterTextData in IdentifiedInformationInChapterTextDataList)
{
var transformedRdfKnowledge =
_transformerServiceHelper.GetTestDataForTransformNlpInformationToRdfKnowledge(
identifiedInformationInChapterTextData);
TransformedRdfKnowledgeList.Add(transformedRdfKnowledge);
transformerCounter++;
_logger.LogInformation($"{Environment.NewLine}Transformation {transformerCounter}. Success!{Environment.NewLine}");
}
}
else // send request to Transformer Service API
{
_logger.LogInformation($"{Environment.NewLine}Asking Transformer Service to transform information into knowledge.{Environment.NewLine}");
var transformerCounter = 0;
foreach (var identifiedInformationInChapterTextData in IdentifiedInformationInChapterTextDataList)
{
var transformedRdfKnowledge =
await _transformerServiceApi.TransformNlpInformationToRdfKnowledge(
_transformerServiceSettings.ApiBaseUrl,
identifiedInformationInChapterTextData);
TransformedRdfKnowledgeList.Add(transformedRdfKnowledge);
transformerCounter++;
_logger.LogInformation($"{Environment.NewLine}Transformation {transformerCounter}. Success!{Environment.NewLine}");
}
}
//
// Authentication Service
//
try
{
_logger.LogInformation($"{Environment.NewLine}Asking Authentication Service for access token.{Environment.NewLine}");
TopBraidEdgOAuthAccessToken = await _authenticationApi.GetAuthenticationToken(
_authenticationServiceSettings.ApiBaseUrl,
_authenticationServiceSettings.ClientId,
_authenticationServiceSettings.ClientSecret,
_authenticationServiceSettings.Scope);
_logger.LogInformation($"{Environment.NewLine}Successfully loaded access token.{Environment.NewLine}");
}
catch (Exception e)
{
_logger.LogInformation($"{Environment.NewLine}Oh no.. :-O Something went wrong. Here is an error message:{Environment.NewLine}");
_logger.LogInformation(e.ToString());
_logger.LogInformation($"{Environment.NewLine}Service Controller application ended.{Environment.NewLine}");
return;
}
//
// Knowledge Service
//
foreach (var transformedRdfKnowledge in TransformedRdfKnowledgeList)
{
_logger.LogInformation($"{Environment.NewLine}Asking Knowledge Service to construct SPARQL INSERT query.{Environment.NewLine}");
var topBraidEdgSparqlInsertBuilder = new Entities.KnowledgeService.TopBraidEdgSparqlInsertBuilder(
_knowledgeServiceSettings.TopBraidEdgOntologyId,
_knowledgeServiceSettings.TopBraidEdgWorkflowId,
_knowledgeServiceSettings.TopBraidEdgUserId,
transformedRdfKnowledge
);
_logger.LogInformation($"{Environment.NewLine}Successfully parsed {topBraidEdgSparqlInsertBuilder.Graph.Nodes.Count()} triples from Transformer Service.{Environment.NewLine}");
var sparqlInsertQueryString = topBraidEdgSparqlInsertBuilder.BuildSparqlInsertQueryString();
_logger.LogInformation($"{Environment.NewLine}Successfully constructed SPARQL INSERT query.{Environment.NewLine}");
var topBraidEdgGraphUrn = topBraidEdgSparqlInsertBuilder.BuildTopBraidEdgGraphUrn();
_logger.LogInformation($"{Environment.NewLine}Loading knowledge into TopBraid EDG graph: {topBraidEdgGraphUrn}{Environment.NewLine}");
try
{
await _topBraidEdgApi.TestInsert(
_knowledgeServiceSettings.ApiBaseUrl,
TopBraidEdgOAuthAccessToken,
sparqlInsertQueryString,
topBraidEdgGraphUrn);
}
catch (Exception e)
{
_logger.LogInformation($"{Environment.NewLine}Oh no.. :-O Something went wrong. Here is an error message:{Environment.NewLine}");
_logger.LogInformation(e.ToString());
_logger.LogInformation($"{Environment.NewLine}Service Controller application ended.{Environment.NewLine}");
return;
}
_logger.LogInformation($"{Environment.NewLine}Successfully loaded knowledge.{Environment.NewLine}");
}
_logger.LogInformation($"{Environment.NewLine}Queued Background Task (completed): {requestedTextServiceRegulationIri}{Environment.NewLine}");
}
static async Task <int> Main(string[] args)
{
var hostBuilder = new HostBuilder()
.ConfigureAppConfiguration((hostContext, configurationBuilder) =>
{
configurationBuilder.AddUserSecrets <Program>();
Configuration = configurationBuilder.Build();
//
// Load settings
//
AuthenticationServiceSettings =
Configuration.GetSection("AuthenticationServiceSettings")
.Get <AuthenticationServiceSettings>();
TextServiceSettings =
Configuration.GetSection("TextServiceSettings")
.Get <TextServiceSettings>();
NlpServiceSettings =
Configuration.GetSection("NlpServiceSettings")
.Get <NlpServiceSettings>();
TransformerServiceSettings =
Configuration.GetSection("TransformerServiceSettings")
.Get <TransformerServiceSettings>();
KnowledgeServiceSettings =
Configuration.GetSection("KnowledgeServiceSettings")
.Get <KnowledgeServiceSettings>();
})
.ConfigureServices((hostContext, services) =>
{
services.AddHttpClient();
// api
services.AddTransient <ITextServiceApi, TextServiceApi>();
services.AddTransient <INlpServiceApi, NlpServiceApi>();
services.AddTransient <ITransformerServiceApi, TransformerServiceApi>();
services.AddTransient <ITopBraidEdgApi, TopBraidEdgApi>();
services.AddTransient <IAuthenticationApi, AuthenticationApi>();
// helpers
services.AddTransient <ITextServiceHelper, TextServiceHelper>();
services.AddTransient <INlpServiceHelper, NlpServiceHelper>();
})
.UseConsoleLifetime();
var host = hostBuilder.Build();
using var serviceScope = host.Services.CreateScope();
{
var services = serviceScope.ServiceProvider;
// api
var textServiceApi = services.GetRequiredService <ITextServiceApi>();
var nlpServiceApi = services.GetRequiredService <INlpServiceApi>();
var transformerServiceApi = services.GetRequiredService <ITransformerServiceApi>();
var topBraidEdgApi = services.GetRequiredService <ITopBraidEdgApi>();
var authenticationApi = services.GetRequiredService <IAuthenticationApi>();
// helpers
var textServiceHelper = services.GetRequiredService <ITextServiceHelper>();
var nlpServiceHelper = services.GetRequiredService <INlpServiceHelper>();
Console.ResetColor();
Console.WriteLine("Service Controller application started.");
//
// Text Service
//
// Get regulation options
var regulationListFromTextService =
await textServiceApi.GetRegulationList(TextServiceSettings.ApiBaseUrl);
var regulationDictionary =
textServiceHelper.MapRegulationResources(regulationListFromTextService);
do
{
// Print regulation options
var regulationDictionaryPrinter = new TextServiceRegulationDictionaryPrinter(regulationDictionary);
regulationDictionaryPrinter.PrintAllOptions();
// Question to user
Console.Write($"Please select a regulation from the list ({1}-{regulationDictionary.Count}):");
var regulationRequestedByTheUser = Console.ReadLine();
var selectedRegulationDictionaryNumber = Convert.ToInt32(regulationRequestedByTheUser);
// Get selected regulation
var selectedRegulationDictionary = regulationDictionary[selectedRegulationDictionaryNumber];
Console.ForegroundColor = ConsoleColor.Black;
Console.BackgroundColor = ConsoleColor.Blue;
Console.WriteLine(string.Format(
"Asking Text Service for regulation {0}-{1}-{2}-{3} ({4}).",
selectedRegulationDictionary.RegulationYear,
selectedRegulationDictionary.RegulationMonth,
selectedRegulationDictionary.RegulationDay,
selectedRegulationDictionary.RegulationNumber,
selectedRegulationDictionary.Language));
var regulationFromTextService = await textServiceApi.GetRegulation(
TextServiceSettings.ApiBaseUrl,
Convert.ToInt32(selectedRegulationDictionary.RegulationYear),
Convert.ToInt32(selectedRegulationDictionary.RegulationMonth),
Convert.ToInt32(selectedRegulationDictionary.RegulationDay),
Convert.ToInt32(selectedRegulationDictionary.RegulationNumber));
var chapterList = textServiceHelper.SplitRegulationResponseIntoChapterList(regulationFromTextService);
Console.WriteLine($"{chapterList.Count} chapters loaded successfully.");
Console.ResetColor();
//
// NLP Service
//
// Get NLP options
var nlpResourceListFromNlpService = await nlpServiceApi.GetNlpResourceList(NlpServiceSettings.ApiBaseUrl);
var nlpResourceDictionary = nlpServiceHelper.MapNlpResources(NlpServiceSettings.ApiBaseUrl, nlpResourceListFromNlpService);
// Print NLP options
var nlpResourceDictionaryPrinter = new NlpServiceResourceDictionaryPrinter(nlpResourceDictionary);
nlpResourceDictionaryPrinter.PrintAllOptions();
// Question to user
Console.Write($"Please select a NLP resource from the list ({1}-{nlpResourceDictionary.Count}):");
var nlpResourceRequestedByTheUser = Console.ReadLine();
var selectedNlpResourceDictionaryNumber = Convert.ToInt32(nlpResourceRequestedByTheUser);
var selectedNlpResourceDictionary = nlpResourceDictionary[selectedNlpResourceDictionaryNumber];
// Send requests to NLP service
Console.BackgroundColor = ConsoleColor.Green;
Console.ForegroundColor = ConsoleColor.Black;
Console.WriteLine(string.Format(
"Asking NLP Service to identify information about {0} ({1}) in regulation {2}-{3}-{4}-{5} ({6}).",
selectedNlpResourceDictionary.Title,
selectedNlpResourceDictionary.Language,
selectedRegulationDictionary.RegulationYear,
selectedRegulationDictionary.RegulationMonth,
selectedRegulationDictionary.RegulationDay,
selectedRegulationDictionary.RegulationNumber,
selectedRegulationDictionary.Language));
Console.ResetColor();
var totalNlpFindingCounter = 0;
for (var i = 0; i < chapterList.Count; i++)
{
Console.BackgroundColor = ConsoleColor.Yellow;
Console.ForegroundColor = ConsoleColor.Black;
var requestNumber = i + 1;
Console.WriteLine($"Processing chapter {requestNumber} of {chapterList.Count}:");
Console.ResetColor();
var identifiedInformationInChapterTextData =
await nlpServiceApi.IdentifyInformationInChapterTextData(
chapterList[i],
selectedNlpResourceDictionary.Url);
var itemCountOfNlpServiceResponse =
nlpServiceHelper.CountItemsInNlpServiceApiResponse(
identifiedInformationInChapterTextData);
if (itemCountOfNlpServiceResponse > 0)
{
Console.BackgroundColor = ConsoleColor.Green;
Console.ForegroundColor = ConsoleColor.Black;
Console.WriteLine($"{itemCountOfNlpServiceResponse} detections in this chapter:");
Console.ResetColor();
var nlpServiceResultPrinter = new NlpServiceResultPrinter(identifiedInformationInChapterTextData);
nlpServiceResultPrinter.PrintAllItems();
totalNlpFindingCounter += itemCountOfNlpServiceResponse;
IdentifiedInformationInChapterTextDataList.Add(identifiedInformationInChapterTextData);
}
else
{
Console.BackgroundColor = ConsoleColor.Green;
Console.ForegroundColor = ConsoleColor.Black;
Console.WriteLine("No detections in this chapter.");
Console.ResetColor();
}
}
if (totalNlpFindingCounter > 0)
{
// Question to user:
Console.Write($"Would you like to proceed with the {totalNlpFindingCounter} NLP findings or start over? Continue (c) or Start over (s): ");
var userInput = Console.ReadLine();
var userInputToLower = userInput?.ToLower();
if (userInputToLower == "c")
{
break;
}
}
else
{
Console.Write("There were no NLP findings. Please press enter to start over.");
Console.ReadLine();
Console.Write("Resetting memory.");
IdentifiedInformationInChapterTextDataList = new List <JsonElement>();
}
Console.WriteLine("Service Controller application is starting over.");
} while (true);
//
// Transformer Service
//
Console.ForegroundColor = ConsoleColor.Black;
Console.BackgroundColor = ConsoleColor.Magenta;
Console.WriteLine("Asking Transformer Service to transform information.");
foreach (var identifiedInformationInChapterTextData in IdentifiedInformationInChapterTextDataList)
{
var transformedRdfKnowledge =
await transformerServiceApi.TransformNlpInformationToRdfKnowledge(
TransformerServiceSettings.ApiBaseUrl,
identifiedInformationInChapterTextData);
TransformedRdfKnowledgeList.Add(transformedRdfKnowledge);
}
Console.ResetColor();
//
// Authentication Service
//
try
{
Console.ForegroundColor = ConsoleColor.Black;
Console.BackgroundColor = ConsoleColor.DarkCyan;
Console.WriteLine("Asking Authentication Service for access token.");
TopBraidEdgOAuthAccessToken = await authenticationApi.GetAuthenticationToken(
AuthenticationServiceSettings.ApiBaseUrl,
AuthenticationServiceSettings.ClientId,
AuthenticationServiceSettings.ClientSecret,
AuthenticationServiceSettings.Scope);
Console.WriteLine("Successfully loaded access token.");
Console.ResetColor();
}
catch (Exception e)
{
Console.ForegroundColor = ConsoleColor.Black;
Console.BackgroundColor = ConsoleColor.Red;
Console.WriteLine("Oh no.. :-O Something went wrong. Here is an error message:");
Console.WriteLine(e);
Console.ResetColor();
Console.WriteLine("Service Controller application ended.");
return(0);
}
//
// Knowledge Service
//
foreach (var transformedRdfKnowledge in TransformedRdfKnowledgeList)
{
Console.ForegroundColor = ConsoleColor.Black;
Console.BackgroundColor = ConsoleColor.Cyan;
Console.WriteLine("Asking Knowledge Service to construct SPARQL INSERT query.");
var topBraidEdgSparqlInsertBuilder = new Entities.KnowledgeService.TopBraidEdgSparqlInsertBuilder(
KnowledgeServiceSettings.TopBraidEdgOntologyId,
KnowledgeServiceSettings.TopBraidEdgWorkflowId,
KnowledgeServiceSettings.TopBraidEdgUserId,
transformedRdfKnowledge
);
Console.WriteLine($"Successfully parsed {topBraidEdgSparqlInsertBuilder.Graph.Nodes.Count()} triples from Transformer Service.");
var sparqlInsertQueryString = topBraidEdgSparqlInsertBuilder.BuildSparqlInsertQueryString();
Console.WriteLine("Successfully constructed SPARQL INSERT query.");
var topBraidEdgGraphUrn = topBraidEdgSparqlInsertBuilder.BuildTopBraidEdgGraphUrn();
Console.WriteLine($"Loading knowledge into TopBraid EDG graph: {topBraidEdgGraphUrn}");
try
{
await topBraidEdgApi.TestInsert(
KnowledgeServiceSettings.ApiBaseUrl,
TopBraidEdgOAuthAccessToken,
sparqlInsertQueryString,
topBraidEdgGraphUrn);
}
catch (Exception e)
{
Console.ForegroundColor = ConsoleColor.Black;
Console.BackgroundColor = ConsoleColor.Red;
Console.WriteLine("Oh no.. :-O Something went wrong. Here is an error message:");
Console.WriteLine(e);
Console.ResetColor();
Console.WriteLine("Service Controller application ended.");
return(0);
}
Console.ForegroundColor = ConsoleColor.Black;
Console.BackgroundColor = ConsoleColor.Cyan;
Console.WriteLine($"Successfully loaded knowledge. Please visit TopBraid EDG (workflow: {topBraidEdgSparqlInsertBuilder.WorkflowId}) to review the result.");
}
Console.ResetColor();
Console.WriteLine("Service Controller application ended.");
}
return(0);
}