public async Task <List <string> > GetInputsToAnalyseAsync(string pathToFile)
{
int totalLines = 0;
int totalInputs = 0;
int totalDistincts = 0;
var inputsToAnalyse = new List <string>();
using (var reader = new StreamReader(pathToFile, detectEncodingFromByteOrderMarks: true))
{
var line = "";
while ((line = (await reader.ReadLineAsync())) != null)
{
totalLines++;
line = line.Trim();
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
totalInputs++;
inputsToAnalyse.Add(line);
}
}
inputsToAnalyse = inputsToAnalyse.Distinct().ToList();
totalDistincts = inputsToAnalyse.Count;
_logger.LogDebug($"Distinct/Inputs: {totalDistincts}/{totalInputs} - Total Lines: {totalLines}");
return(inputsToAnalyse);
}
public async Task ExportNLPModelAsync(string authorization, string outputFilePath, string excel = null)
{
if (string.IsNullOrEmpty(authorization))
{
throw new ArgumentNullException(nameof(authorization));
}
if (string.IsNullOrEmpty(outputFilePath))
{
throw new ArgumentNullException(nameof(outputFilePath));
}
var blipAIClient = _blipClientFactory.GetInstanceForAI(authorization);
_logger.LogDebug("NLP Export\n");
var intentions = await blipAIClient.GetAllIntentsAsync();
var entities = await blipAIClient.GetAllEntities();
_fileManagerService.CreateDirectoryIfNotExists(outputFilePath);
if (!string.IsNullOrEmpty(excel))
{
List <NLPExportModel> excelExportModels = new List <NLPExportModel>
{
WriteIntentionExcel(intentions),
WriteQuestionsExcel(intentions),
WriteAnswersExcel(intentions),
WriteEntitiesExcel(entities)
};
_excelGeneratorService.WriteContentOnExcel(excelExportModels, outputFilePath, excel);
}
else
{
var csvExportModels = new List <NLPExportModel>
{
WriteIntentionCSV(intentions, outputFilePath),
WriteAnswersCSV(intentions, outputFilePath),
WriteEntitiesCSV(entities, outputFilePath)
};
_csvGeneratorService.WriteContentOnCSV(csvExportModels, outputFilePath);
}
_logger.LogDebug("DONE");
}
public async Task <List <Entity> > GetAllEntities(bool verbose = false)
{
var entitiesList = new List <Entity>();
try
{
var command = new Command
{
Id = EnvelopeId.NewId(),
To = Node.Parse("*****@*****.**"),
Uri = new LimeUri($"/entities"),
Method = CommandMethod.Get,
};
_logger.LogDebug("Start get entities");
var envelopeResult = await RunCommandAsync(command);
var entities = envelopeResult.Resource as DocumentCollection ?? new DocumentCollection {
Items = Enumerable.Empty <Document>().ToArray()
};
var counter = 0;
var totalEntities = entities.Total;
var str = BuildProcessBarLog(totalEntities, counter);
_logger.LogTrace(str.ToString());
foreach (var entity in entities)
{
entitiesList.Add(entity as Entity);
counter++;
str = BuildProcessBarLog(totalEntities, counter);
_logger.LogTrace(str.ToString());
}
_logger.LogDebug("Finish get entities");
return(entitiesList);
}
catch (HttpRequestException e)
{
_logger.LogError(e, "\nException Caught!");
_logger.LogError(e, "Message :{0} ", e.Message);
return(null);
}
}
public async Task AnalyseAsync(string authorization, string inputSource, string reportOutput, bool doContentCheck = false, bool rawContent = false)
{
if (string.IsNullOrEmpty(authorization))
{
throw new ArgumentNullException("You must provide the target bot (node) for this action.");
}
if (string.IsNullOrEmpty(inputSource))
{
throw new ArgumentNullException("You must provide the input source (phrase or file) for this action.");
}
if (string.IsNullOrEmpty(reportOutput))
{
throw new ArgumentNullException("You must provide the full output's report file name for this action.");
}
_logger.LogDebug("COMEÇOU!");
_fileService.CreateDirectoryIfNotExists(reportOutput);
var client = _blipClientFactory.GetInstanceForAI(authorization);
IContentManagerApiClient contentClient = new ContentManagerApiClient(authorization);
var allIntents = new List <Intention>();
if (doContentCheck)
{
_logger.LogDebug("\tCarregando intencoes...");
allIntents = await client.GetAllIntentsAsync();
_logger.LogDebug("\tCarregadas!");
}
var inputType = InputType.Phrase;
inputType = DetectInputType(inputSource);
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = DataflowBlockOptions.Unbounded,
MaxDegreeOfParallelism = 20,
};
var analyseBlock = new TransformBlock <DataBlock, DataBlock>((Func <DataBlock, Task <DataBlock> >)AnalyseForMetrics, options);
var checkBlock = new TransformBlock <DataBlock, DataBlock>((Func <DataBlock, DataBlock>)CheckResponse, options);
var contentBlock = new TransformBlock <DataBlock, DataBlock>((Func <DataBlock, Task <DataBlock> >)GetContent, options);
var buildResultBlock = new ActionBlock <DataBlock>(BuildResult, new ExecutionDataflowBlockOptions
{
BoundedCapacity = DataflowBlockOptions.Unbounded,
MaxMessagesPerTask = 1
});
analyseBlock.LinkTo(checkBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
checkBlock.LinkTo(contentBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
contentBlock.LinkTo(buildResultBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
_count = 0;
var inputList = await GetInputList(inputType, inputSource, client, contentClient, reportOutput, allIntents, doContentCheck, rawContent);
_total = inputList.Count;
foreach (var input in inputList)
{
await analyseBlock.SendAsync(input);
}
analyseBlock.Complete();
await buildResultBlock.Completion;
_logger.LogDebug("TERMINOU!");
}
public async Task AnalyseAsync(string authorization, string inputSource, string reportOutput, bool doContentCheck = false)
{
if (string.IsNullOrEmpty(authorization))
{
throw new ArgumentNullException("You must provide the target bot (node) for this action.");
}
if (string.IsNullOrEmpty(inputSource))
{
throw new ArgumentNullException("You must provide the input source (phrase or file) for this action.");
}
if (string.IsNullOrEmpty(reportOutput))
{
throw new ArgumentNullException("You must provide the full output's report file name for this action.");
}
_logger.LogDebug("COMEÇOU!");
_fileService.CreateDirectoryIfNotExists(reportOutput);
var bucketStorage = new BucketStorage("Key " + authorization);
var contentProvider = new Take.ContentProvider.ContentProvider(bucketStorage, 5);
var client = _blipClientFactory.GetInstanceForAI(authorization);
var allIntents = new List <Intention>();
if (doContentCheck)
{
_logger.LogDebug("\tCarregando intencoes...");
allIntents = await client.GetAllIntentsAsync();
_logger.LogDebug("\tCarregadas!");
}
bool isPhrase = false;
var isDirectory = _fileService.IsDirectory(inputSource);
var isFile = _fileService.IsFile(inputSource);
if (isFile)
{
_logger.LogDebug("\tA entrada é um arquivo");
isPhrase = false;
}
else
if (isDirectory)
{
_logger.LogError("\tA entrada é um diretório");
throw new ArgumentNullException("You must provide the input source (phrase or file) for this action. Your input was a direcory.");
}
else
{
_logger.LogDebug("\tA entrada é uma frase");
isPhrase = true;
}
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = DataflowBlockOptions.Unbounded,
MaxDegreeOfParallelism = 20,
};
var analyseBlock = new TransformBlock <NLPAnalyseDataBlock, NLPAnalyseDataBlock>((Func <NLPAnalyseDataBlock, Task <NLPAnalyseDataBlock> >)AnalyseForMetrics, options);
var checkBlock = new TransformBlock <NLPAnalyseDataBlock, NLPAnalyseDataBlock>((Func <NLPAnalyseDataBlock, NLPAnalyseDataBlock>)CheckResponse, options);
var contentBlock = new TransformBlock <NLPAnalyseDataBlock, NLPAnalyseDataBlock>((Func <NLPAnalyseDataBlock, Task <NLPAnalyseDataBlock> >)GetContent, options);
var showResultBlock = new ActionBlock <NLPAnalyseDataBlock>(BuildResult, new ExecutionDataflowBlockOptions
{
BoundedCapacity = DataflowBlockOptions.Unbounded,
MaxMessagesPerTask = 1
});
analyseBlock.LinkTo(checkBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
checkBlock.LinkTo(contentBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
contentBlock.LinkTo(showResultBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
_count = 0;
var inputList = await GetInputList(isPhrase, inputSource, client, reportOutput, allIntents, contentProvider, doContentCheck);
_total = inputList.Count;
foreach (var input in inputList)
{
await analyseBlock.SendAsync(input);
}
analyseBlock.Complete();
await showResultBlock.Completion;
_logger.LogDebug("TERMINOU!");
}