private async Task <List <TestingExample> > CreateTestData(CustomTextGetLabeledExamplesResponse labeledExamples, ConcurrentBag <string> convertedFiles, ConcurrentDictionary <string, string> failedFiles)
{
var modelsDictionary = await _customTextAuthoringService.GetModelsDictionary();
var testingExamples = new List <TestingExample>();
await _destinationStorageService.CreateDirectoryAsync(Constants.EvaluationCommandPredictionOutputDirectoryName);
foreach (var labeledExample in labeledExamples.Examples)
{
try
{
// document text
var documentText = await _sourceStorageService.ReadFileAsStringAsync(labeledExample.Document.DocumentId);
// prediction
_loggerService.LogOperation(OperationType.RunningPrediction, labeledExample.Document.DocumentId);
var predictionResponse = await _customTextPredictionService.GetPredictionAsync(documentText);
// store prediction output
var predictionResponseString = JsonConvert.SerializeObject(predictionResponse, Formatting.Indented);
var jsonFileName = Path.GetFileNameWithoutExtension(labeledExample.Document.DocumentId) + ".json";
await _destinationStorageService.StoreDataToDirectoryAsync(predictionResponseString, Constants.EvaluationCommandPredictionOutputDirectoryName, jsonFileName);
// create test example
var testExample = BatchTestingInputMapper.CreateTestExample(documentText, labeledExample, predictionResponse, modelsDictionary);
testingExamples.Add(testExample);
convertedFiles.Add(labeledExample.Document.DocumentId);
}
catch (Exception ex)
{
failedFiles[labeledExample.Document.DocumentId] = ex.Message;
_loggerService.LogError(ex);
}
}
return(testingExamples);
}
public async Task EvaluateCustomTextAppAsync(StorageType sourceStorageType, StorageType destinationStorageType)
{
InitializeStorage(sourceStorageType, destinationStorageType);
// init result containers
var convertedFiles = new ConcurrentBag <string>();
var failedFiles = new ConcurrentDictionary <string, string>();
// map app models to nuget classes
var customTextModels = await _customTextAuthoringService.GetApplicationModels();
var mappedEntities = BatchTestingInputMapper.MapCustomTextAppEntityModels(customTextModels.Models, entityPrefix: string.Empty);
var mappedClasses = BatchTestingInputMapper.MapCustomTextAppClassModels(customTextModels.Models);
// get labeled examples
_loggerService.LogOperation(OperationType.GeneratingTestSet);
var labeledExamples = await _customTextAuthoringService.GetLabeledExamples();
var mappedTestData = await CreateTestData(labeledExamples, convertedFiles, failedFiles);
// evaluate model
_loggerService.LogOperation(OperationType.EvaluatingResults);
var batchTestResponse = _batchTestingService.RunBatchTest(mappedTestData, mappedEntities, mappedClasses);
// map output data
var mappedBatchTestResponse = BatchTestingOutputMapper.MapEvaluationOutput(batchTestResponse);
// store file
var outFileName = Constants.CustomTextEvaluationControllerOutputFileName;
_loggerService.LogOperation(OperationType.StoringResult, outFileName);
var responseAsJson = JsonConvert.SerializeObject(mappedBatchTestResponse, Formatting.Indented);
await _destinationStorageService.StoreDataAsync(responseAsJson, outFileName);
// log result
_loggerService.LogParsingResult(convertedFiles, failedFiles);
}
