Пример #1
0
        private async Task <List <TestingExample> > CreateTestData(CustomTextGetLabeledExamplesResponse labeledExamples, ConcurrentBag <string> convertedFiles, ConcurrentDictionary <string, string> failedFiles)
        {
            var modelsDictionary = await _customTextAuthoringService.GetModelsDictionary();

            var testingExamples = new List <TestingExample>();
            await _destinationStorageService.CreateDirectoryAsync(Constants.EvaluationCommandPredictionOutputDirectoryName);

            foreach (var labeledExample in labeledExamples.Examples)
            {
                try
                {
                    // document text
                    var documentText = await _sourceStorageService.ReadFileAsStringAsync(labeledExample.Document.DocumentId);

                    // prediction
                    _loggerService.LogOperation(OperationType.RunningPrediction, labeledExample.Document.DocumentId);
                    var predictionResponse = await _customTextPredictionService.GetPredictionAsync(documentText);

                    // store prediction output
                    var predictionResponseString = JsonConvert.SerializeObject(predictionResponse, Formatting.Indented);
                    var jsonFileName             = Path.GetFileNameWithoutExtension(labeledExample.Document.DocumentId) + ".json";
                    await _destinationStorageService.StoreDataToDirectoryAsync(predictionResponseString, Constants.EvaluationCommandPredictionOutputDirectoryName, jsonFileName);

                    // create test example
                    var testExample = BatchTestingInputMapper.CreateTestExample(documentText, labeledExample, predictionResponse, modelsDictionary);
                    testingExamples.Add(testExample);


                    convertedFiles.Add(labeledExample.Document.DocumentId);
                }
                catch (Exception ex)
                {
                    failedFiles[labeledExample.Document.DocumentId] = ex.Message;
                    _loggerService.LogError(ex);
                }
            }
            return(testingExamples);
        }
Пример #2
0
        public async Task EvaluateCustomTextAppAsync(StorageType sourceStorageType, StorageType destinationStorageType)
        {
            InitializeStorage(sourceStorageType, destinationStorageType);

            // init result containers
            var convertedFiles = new ConcurrentBag <string>();
            var failedFiles    = new ConcurrentDictionary <string, string>();

            // map app models to nuget classes
            var customTextModels = await _customTextAuthoringService.GetApplicationModels();

            var mappedEntities = BatchTestingInputMapper.MapCustomTextAppEntityModels(customTextModels.Models, entityPrefix: string.Empty);
            var mappedClasses  = BatchTestingInputMapper.MapCustomTextAppClassModels(customTextModels.Models);

            // get labeled examples
            _loggerService.LogOperation(OperationType.GeneratingTestSet);
            var labeledExamples = await _customTextAuthoringService.GetLabeledExamples();

            var mappedTestData = await CreateTestData(labeledExamples, convertedFiles, failedFiles);

            // evaluate model
            _loggerService.LogOperation(OperationType.EvaluatingResults);
            var batchTestResponse = _batchTestingService.RunBatchTest(mappedTestData, mappedEntities, mappedClasses);

            // map output data
            var mappedBatchTestResponse = BatchTestingOutputMapper.MapEvaluationOutput(batchTestResponse);

            // store file
            var outFileName = Constants.CustomTextEvaluationControllerOutputFileName;

            _loggerService.LogOperation(OperationType.StoringResult, outFileName);
            var responseAsJson = JsonConvert.SerializeObject(mappedBatchTestResponse, Formatting.Indented);
            await _destinationStorageService.StoreDataAsync(responseAsJson, outFileName);

            // log result
            _loggerService.LogParsingResult(convertedFiles, failedFiles);
        }