async public Task GetDatasetDefinitions_GivenDefinitionsRequestedButContainsNoResults_ReturnsEmptyArray() { //Arrange HttpRequest request = Substitute.For <HttpRequest>(); IEnumerable <DatasetDefinition> definitions = new DatasetDefinition[0]; IDatasetRepository repository = CreateDataSetsRepository(); repository .GetDatasetDefinitions() .Returns(definitions); DefinitionsService service = CreateDefinitionsService(datasetsRepository: repository); //Act IActionResult result = await service.GetDatasetDefinitions(request); //Assert result .Should() .BeOfType <OkObjectResult>(); OkObjectResult objResult = (OkObjectResult)result; IEnumerable <DatasetDefinition> objValue = (IEnumerable <DatasetDefinition>)objResult.Value; objValue .Count() .Should() .Be(0); }
public async Task <IActionResult> GetDatasetSchemaSasUrl(HttpRequest request) { string json = await request.GetRawBodyStringAsync(); DatasetSchemaSasUrlRequestModel requestModel = JsonConvert.DeserializeObject <DatasetSchemaSasUrlRequestModel>(json); if (requestModel == null) { _logger.Warning("No dataset schema request model was provided"); return(new BadRequestObjectResult("No dataset schema request model was provided")); } if (requestModel.DatasetDefinitionId.IsNullOrEmpty()) { _logger.Warning("No dataset schema name was provided"); return(new BadRequestObjectResult("No dataset schema name was provided")); } DatasetDefinition datasetDefinition = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(requestModel.DatasetDefinitionId)); if (datasetDefinition == null) { return(new NotFoundObjectResult("Data schema definiton not found")); } string definitionName = datasetDefinition.Name.Replace("/", "_").Replace("\\", "_"); string fileName = $"schemas/{definitionName}.xlsx"; string blobUrl = _blobClient.GetBlobSasUrl(fileName, DateTimeOffset.UtcNow.AddDays(1), SharedAccessBlobPermissions.Read); return(new OkObjectResult(new DatasetSchemaSasUrlResponseModel { SchemaUrl = blobUrl })); }
public DatasetUploadValidationModel(ExcelPackage excelPackage, Func <IEnumerable <ProviderSummary> > providerSummaries, DatasetDefinition datasetDefinition) { ExcelPackage = excelPackage; ProviderSummaries = providerSummaries; DatasetDefinition = datasetDefinition; ValidationResult = new DatasetUploadValidationResult(); }
public void Validate_GivenDuplicateDatasetDefinitionName_ValidIsFalse() { //Arrange DatasetDefinition model = CreateModel(); model.Name = DatasetDefinitionName; model.Id = DatasetDefinitionId; DatasetDefinitionValidator validator = CreateValidator(); //Act ValidationResult result = validator.Validate(model); //Assert result .IsValid .Should() .BeFalse(); result .Errors .Count .Should() .Be(1); }
private void AssertDatasetIsPopulated(DatasetDefinition dataset) { Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.DatasetName)); Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.DisplayName)); Assert.IsNotEmpty(dataset.IndexBy); if (UseMocks) { Assert.IsNotNull(dataset.LastScheduledDate); Assert.IsNotNull(dataset.LatestSubtimeframeAvailable); } Assert.IsNotNull(dataset.Query); Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.ProjectId)); Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.AnalysisType)); Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.EventCollection)); Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.Timeframe)); Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.Interval)); // TODO : We'll need to do some setup to actually get this to run automatically // with !UseMocks...and take into account that it can "take up to an hour for a newly // created dataset to compute results for the first time." Assert.IsNotNull(dataset.Query.GroupBy); Assert.IsTrue(dataset.Query.GroupBy.Count() == 1); if (dataset.Query.Filters != null) { foreach (var filter in dataset.Query.Filters) { AssertFilterIsPopulated(filter); } } }
public DatasetDefinitionChanges DetectChanges(DatasetDefinition newDatasetDefinition, DatasetDefinition existingDatasetDefinition) { Guard.ArgumentNotNull(newDatasetDefinition, nameof(newDatasetDefinition)); Guard.ArgumentNotNull(existingDatasetDefinition, nameof(existingDatasetDefinition)); DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges { Id = newDatasetDefinition.Id }; string newDatasetDefinitionAsJson = JsonConvert.SerializeObject(newDatasetDefinition); string existingDatasetDefinitionAsJson = JsonConvert.SerializeObject(existingDatasetDefinition); if (string.Equals(newDatasetDefinitionAsJson, existingDatasetDefinitionAsJson)) { return(datasetDefinitionChanges); } if (newDatasetDefinition.Name != existingDatasetDefinition.Name) { datasetDefinitionChanges.DefinitionChanges.Add(DefinitionChangeType.DefinitionName); datasetDefinitionChanges.NewName = newDatasetDefinition.Name; } datasetDefinitionChanges.TableDefinitionChanges.AddRange(DetectTableDefinitionChanges(existingDatasetDefinition.TableDefinitions, newDatasetDefinition.TableDefinitions)); return(datasetDefinitionChanges); }
public async Task <HttpStatusCode> UpsertDatasetDefinition(DatasetDefinition definition) { Guard.ArgumentNotNull(definition, nameof(definition)); return(await PostAsync($"{UrlRoot}/datasetdefinition", definition)); }
private async Task <TableLoadResult> GetTableResult(string fullBlobName, DatasetDefinition datasetDefinition) { string dataset_cache_key = $"{CacheKeys.DatasetRows}:{datasetDefinition.Id}:{GetBlobNameCacheKey(fullBlobName)}".ToLowerInvariant(); IEnumerable <TableLoadResult> tableLoadResults = await _cacheProvider.GetAsync <TableLoadResult[]>(dataset_cache_key); if (tableLoadResults.IsNullOrEmpty()) { ICloudBlob blob = await _blobClient.GetBlobReferenceFromServerAsync(fullBlobName); if (blob == null) { _logger.Error($"Failed to find blob with path: {fullBlobName}"); throw new NonRetriableException($"Failed to find blob with path: {fullBlobName}"); } using (Stream datasetStream = await _blobClient.DownloadToStreamAsync(blob)) { if (datasetStream == null || datasetStream.Length == 0) { _logger.Error($"Invalid blob returned: {fullBlobName}"); throw new NonRetriableException($"Invalid blob returned: {fullBlobName}"); } tableLoadResults = _excelDatasetReader.Read(datasetStream, datasetDefinition).ToList(); } await _cacheProvider.SetAsync(dataset_cache_key, tableLoadResults.ToArraySafe(), TimeSpan.FromDays(7), true); } return(tableLoadResults.FirstOrDefault()); }
public async Task <IActionResult> CreateOrUpdateDatasetDefinition(CreateDatasetDefinitionFromTemplateModel model, string correlationId, Reference user) { ValidationResult validationResult = await _createDatasetDefinitionFromTemplateValidator.ValidateAsync(model); if (!validationResult.IsValid) { string errorMessage = string.Join(";", validationResult.Errors.Select(x => x.ErrorMessage)); _logger.Error(errorMessage); return(new BadRequestObjectResult(errorMessage)); } ; FundingStream fundingStream = await _policyRepository.GetFundingStream(model.FundingStreamId); TemplateMetadataDistinctCalculationsContents templateContents = await _policyRepository.GetDistinctTemplateMetadataCalculationsContents(model.FundingStreamId, model.FundingPeriodId, model.TemplateVersion); if (templateContents == null) { return(new BadRequestObjectResult($"No funding template for given FundingStreamId " + $"- {model.FundingStreamId}, FundingPeriodId - {model.FundingPeriodId}, TemplateVersion - {model.TemplateVersion}")); } DatasetDefinition datasetDefinition = CreateDatasetDefinition(model, templateContents, fundingStream); return(await SaveDatasetDefinition(datasetDefinition, correlationId, user)); }
private static IEnumerable <string> GetProviderIdsForIdentifier(DatasetDefinition datasetDefinition, RowLoadResult row, IEnumerable <ProviderSummary> providerSummaries) { IEnumerable <FieldDefinition> identifierFields = datasetDefinition.TableDefinitions?.First().FieldDefinitions.Where(x => x.IdentifierFieldType.HasValue); foreach (FieldDefinition field in identifierFields) { if (!string.IsNullOrWhiteSpace(field.Name)) { if (row.Fields.ContainsKey(field.Name)) { string identifier = row.Fields[field.Name]?.ToString(); if (!string.IsNullOrWhiteSpace(identifier)) { Dictionary <string, List <string> > lookup = GetDictionaryForIdentifierType(field.IdentifierFieldType, identifier, providerSummaries); if (lookup.TryGetValue(identifier, out List <string> providerIds)) { return(providerIds); } } else { // For debugging only //_logger.Debug("Found identifier with null or emtpy string for provider"); } } } } return(new string[0]); }
public void DetectChanges_GivenTableDefinitionNameChange_ReturnsModelWithChanges() { //Arrange DatasetDefinition newDefinition = new DatasetDefinition { Name = "name2", TableDefinitions = new List <TableDefinition> { new TableDefinition { Name = "name2", FieldDefinitions = new List <FieldDefinition>() } } }; DatasetDefinition existingDefinition = new DatasetDefinition { Name = "name2", TableDefinitions = new List <TableDefinition> { new TableDefinition { Name = "name1", FieldDefinitions = new List <FieldDefinition>() } } }; DefinitionChangesDetectionService service = new DefinitionChangesDetectionService(); //Act DatasetDefinitionChanges changes = service.DetectChanges(newDefinition, existingDefinition); //Assert changes .HasChanges .Should() .BeTrue(); changes .TableDefinitionChanges .First() .HasChanges .Should() .BeTrue(); changes .TableDefinitionChanges .First() .ChangeTypes .First() .Should() .Be(TableDefinitionChangeType.DefinitionName); changes .TableDefinitionChanges .First() .TableDefinition .Name .Should() .Be("name2"); }
public async Task UpsertDatasetDefinitionDelegatesToGraphRepository() { DatasetDefinition definition = NewDataDefinition(); await _datasetRepository.UpsertDatasetDefinition(definition); await ThenTheNodeWasCreated(definition, AttributeConstants.DatasetDefinitionId); }
private static bool UploadDataset(string subscriptionKey, string hostURI, DatasetDefinition datasetDefinition, string wavePath, string scriptPath) { string waveName = Path.GetFileName(wavePath); string scriptName = Path.GetFileName(scriptPath); using (FileStream fsscript = new FileStream(scriptPath, FileMode.Open)) using (FileStream fswave = new FileStream(wavePath, FileMode.Open)) using (var client = new HttpClient()) using (var content = new MultipartFormDataContent()) { client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey); content.Add(new StringContent(datasetDefinition.Name), "name"); if (datasetDefinition.Description != null) { content.Add(new StringContent(datasetDefinition.Description), "description"); } if (datasetDefinition.ProjectId != null) { content.Add(new StringContent(datasetDefinition.ProjectId), "projectId"); } content.Add(new StringContent(datasetDefinition.DataImportKind), "dataImportKind"); content.Add(new StringContent(datasetDefinition.Locale), "locale"); if (datasetDefinition.Properties != null) { content.Add(new StringContent(JsonConvert.SerializeObject(datasetDefinition.Properties)), "properties"); } var transcriptionContent = new StreamContent(fsscript); transcriptionContent.Headers.Add("Content-Disposition", $@"form-data; name=""transcriptions""; filename=""{scriptName}"""); transcriptionContent.Headers.Add("Content-Type", "text/plain"); transcriptionContent.Headers.Add("Content-Length", $"{fsscript.Length}"); content.Add(transcriptionContent, "transcriptions", scriptName); var wavesContent = new StreamContent(fswave); wavesContent.Headers.Add("Content-Disposition", $@"form-data; name=""audiodata""; filename=""{waveName}"""); wavesContent.Headers.Add("Content-Type", "application/x-zip-compressed"); wavesContent.Headers.Add("Content-Length", $"{fswave.Length}"); content.Add(wavesContent, "audiodata", waveName); string url = string.Format(CultureInfo.InvariantCulture, hostURI + API_V3.VoiceDatasets_Upload); var response = client.PostAsync(url, content).Result; if (response.StatusCode != HttpStatusCode.Accepted) { APIHelper.PrintErrorMessage(response); return(false); } System.Console.WriteLine(response.Headers.Location); return(true); } }
//Create Long Audio Dataset public void UploadLongAudioDataset(string waveZipUpload, string scriptZipUpload, string name, string description, string locale, string gender) { var properties = new Dictionary <string, string> { { "Gender", gender } }; var datasetDefinition = new DatasetDefinition(name, description, locale, properties, "CustomVoice"); var submitResponse = VoiceAPIHelper.SubmitLongAudioDataset(datasetDefinition, waveZipUpload, scriptZipUpload, CreateLongAudioDatasetUrl, this.subscriptionKey); }
public TableLoadResultWithHeaders Read(ExcelPackage excelPackage, DatasetDefinition datasetDefinition, bool parse) { if (datasetDefinition.TableDefinitions.Count == 1 && excelPackage.Workbook.Worksheets.Count > 0) { return(ConvertSheetToObjects(excelPackage.Workbook.Worksheets.First(), datasetDefinition.TableDefinitions.First(), parse)); } return(null); }
public async Task Merge_ShouldGetTheNewAndUpdatedData(string latestBlobFileName, string blobFileNameToMerge, string resultsFile, string definitionFileName) { DatasetDefinition datasetDefinition = GetDatasetDefinitionByName(definitionFileName); await using Stream latestDatasetStream = File.OpenRead($"TestItems{Path.DirectorySeparatorChar}{latestBlobFileName}"); Mock <ICloudBlob> latestFileBlob = new Mock <ICloudBlob>(); _blobClient.Setup(_ => _ .GetBlobReferenceFromServerAsync(latestBlobFileName)) .ReturnsAsync(latestFileBlob.Object); _blobClient.Setup(_ => _ .DownloadToStreamAsync(latestFileBlob.Object)) .ReturnsAsync(latestDatasetStream); await using Stream fileToMergeDatasetStream = File.OpenRead($"TestItems{Path.DirectorySeparatorChar}{blobFileNameToMerge}"); Mock <ICloudBlob> fileToMergeBlob = new Mock <ICloudBlob>(); _blobClient.Setup(_ => _ .GetBlobReferenceFromServerAsync(blobFileNameToMerge)) .ReturnsAsync(fileToMergeBlob.Object); _blobClient.Setup(_ => _ .DownloadToStreamAsync(fileToMergeBlob.Object)) .ReturnsAsync(fileToMergeDatasetStream); await using MemoryStream uploadedStream = new MemoryStream(); fileToMergeBlob.Setup(_ => _.UploadFromStreamAsync(It.IsAny <Stream>())) .Callback <Stream>(_ => { _?.Seek(0, SeekOrigin.Begin); // ReSharper disable once AccessToDisposedClosure _?.CopyTo(uploadedStream); }); DatasetDataMergeResult result = await _service.Merge(datasetDefinition, latestBlobFileName, blobFileNameToMerge); result.TablesMergeResults.Count().Should().Be(1); await using Stream expectedResultStream = File.OpenRead($"TestItems{Path.DirectorySeparatorChar}{resultsFile}"); using ExcelPackage expected = new ExcelPackage(expectedResultStream); using ExcelPackage actual = new ExcelPackage(uploadedStream); ExcelWorksheet expectedWorksheet = expected.Workbook.Worksheets[1]; ExcelWorksheet actualWorksheet = expected.Workbook.Worksheets[1]; for (int i = 1; i <= expectedWorksheet.Dimension.Rows; i++) { for (int j = 1; j <= expectedWorksheet.Dimension.Columns; j++) { actualWorksheet.Cells[i, j].Value.Should().Be(expectedWorksheet.Cells[i, j].Value); } } }
private async Task <IEnumerable <IndexError> > IndexDatasetDefinition(DatasetDefinition definition, PoliciesApiModels.FundingStream fundingStream) { // Calculate hash for model to see if there are changes string modelJson = JsonConvert.SerializeObject(definition); string hashCode = ""; using (SHA256 sha256Generator = SHA256Managed.Create()) { byte[] modelBytes = UTF8Encoding.UTF8.GetBytes(modelJson); foreach (byte hashByte in sha256Generator.ComputeHash(modelBytes)) { hashCode += string.Format("{0:X2}", hashByte); } } DatasetDefinitionIndex datasetDefinitionIndex = new DatasetDefinitionIndex() { Id = definition.Id, Name = definition.Name, Description = definition.Description, LastUpdatedDate = DateTimeOffset.Now, ProviderIdentifier = definition.TableDefinitions.FirstOrDefault()?.FieldDefinitions?.Where(f => f.IdentifierFieldType.HasValue)?.Select(f => Enum.GetName(typeof(IdentifierFieldType), f.IdentifierFieldType.Value)).FirstOrDefault(), ModelHash = hashCode, FundingStreamId = fundingStream.Id, FundingStreamName = fundingStream.Name }; if (string.IsNullOrWhiteSpace(datasetDefinitionIndex.ProviderIdentifier)) { datasetDefinitionIndex.ProviderIdentifier = "None"; } bool updateIndex = true; // Only update index if metadata or model has changed, this is to preserve the LastUpdateDate DatasetDefinitionIndex existingIndex = await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() => _datasetDefinitionSearchRepository.SearchById(definition.Id)); if (existingIndex != null) { if (existingIndex.ModelHash == hashCode && existingIndex.Description == definition.Description && existingIndex.Name == definition.Name && existingIndex.ProviderIdentifier == datasetDefinitionIndex.ProviderIdentifier) { updateIndex = false; } } if (updateIndex) { return(await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() => _datasetDefinitionSearchRepository.Index(new DatasetDefinitionIndex[] { datasetDefinitionIndex }))); } return(Enumerable.Empty <IndexError>()); }
private static DatasetDefinition GetDatasetDefinitionByName(string datasetDefinitionName) { JObject obj1 = JObject.Parse(File.ReadAllText($"DatasetDefinitions{Path.DirectorySeparatorChar}{datasetDefinitionName}")); DatasetDefinition datasetDefinition = obj1.ToObject <DatasetDefinition>(); return(datasetDefinition); }
private async Task <BuildProject> GenerateBuildProject(string specificationId) { BuildProject buildproject = new BuildProject { SpecificationId = specificationId, Id = Guid.NewGuid().ToString(), Name = specificationId, DatasetRelationships = new List <DatasetRelationshipSummary>(), Build = new Build() }; IEnumerable <DatasetSpecificationRelationshipViewModel> datasetRelationshipModels = await _datasetRepositoryPolicy.ExecuteAsync(() => _datasetRepository.GetCurrentRelationshipsBySpecificationId(specificationId)); if (!datasetRelationshipModels.IsNullOrEmpty()) { ConcurrentBag <DatasetDefinition> datasetDefinitions = new ConcurrentBag <DatasetDefinition>(); IList <Task> definitionTasks = new List <Task>(); IEnumerable <string> definitionIds = datasetRelationshipModels.Select(m => m.Definition?.Id); foreach (string definitionId in definitionIds) { Task task = Task.Run(async() => { DatasetDefinition datasetDefinition = await _datasetRepositoryPolicy.ExecuteAsync(() => _datasetRepository.GetDatasetDefinitionById(definitionId)); if (datasetDefinition != null) { datasetDefinitions.Add(datasetDefinition); } }); definitionTasks.Add(task); } await TaskHelper.WhenAllAndThrow(definitionTasks.ToArray()); foreach (DatasetSpecificationRelationshipViewModel datasetRelationshipModel in datasetRelationshipModels) { buildproject.DatasetRelationships.Add(new DatasetRelationshipSummary { DatasetDefinitionId = datasetRelationshipModel.Definition.Id, DatasetId = datasetRelationshipModel.DatasetId, Relationship = new Common.Models.Reference(datasetRelationshipModel.Id, datasetRelationshipModel.Name), DefinesScope = datasetRelationshipModel.IsProviderData, Id = datasetRelationshipModel.Id, Name = datasetRelationshipModel.Name, DatasetDefinition = datasetDefinitions.FirstOrDefault(m => m.Id == datasetRelationshipModel.Definition.Id) }); } } return(buildproject); }
public async Task OnGet_WhenSpecificationStatusCodeIsNotFoundThenNotFoundObjectResultReturned() { // Arrange IDatasetsApiClient datasetClient = Substitute.For <IDatasetsApiClient>(); ISpecsApiClient specsClient = Substitute.For <ISpecsApiClient>(); IMapper mapper = MappingHelper.CreateFrontEndMapper(); ILogger logger = Substitute.For <ILogger>(); string expectedSpecificationId = "1"; Specification expectedSpecification = null; AssignDatasetSchemaPageModel datasetSchemaPageModel = CreatePageModel(specsClient, datasetClient, mapper); specsClient .GetSpecification(expectedSpecificationId) .Returns(new ApiResponse <Specification>(HttpStatusCode.NotFound, expectedSpecification)); DatasetDefinition d1 = new DatasetDefinition() { Id = "1", Name = "APT Final Baselines", Description = "Local Authority return of provider baselines", }; DatasetDefinition d2 = new DatasetDefinition() { Id = "2", Name = "School Provider Dataset", Description = "List of providers", }; IEnumerable <DatasetDefinition> dataDefn = new List <DatasetDefinition> { d1, d2 }; datasetClient .GetDataDefinitions() .Returns(new ApiResponse <IEnumerable <DatasetDefinition> >(HttpStatusCode.OK, dataDefn)); // Act IActionResult result = await datasetSchemaPageModel.OnGet(expectedSpecificationId); // Assert result.Should().NotBeNull(); result.Should().BeOfType <NotFoundObjectResult>(); NotFoundObjectResult typeResult = result as NotFoundObjectResult; typeResult.Value.Should().Be("Unable to get specification response. Specification Id value = 1"); Assert.IsInstanceOfType(result, typeof(NotFoundObjectResult)); }
private static DatasetDefinition CreateDatasetDefinition( CreateDatasetDefinitionFromTemplateModel model, TemplateMetadataDistinctCalculationsContents templateContent, FundingStream fundingStream) { int id = model.DatasetDefinitionId; string name = $"{fundingStream.Name}-{model.TemplateVersion}"; DatasetDefinition datasetDefinition = new DatasetDefinition() { Id = id.ToString(), Name = name, Description = name, FundingStreamId = fundingStream.Id }; id += 1; datasetDefinition.TableDefinitions = new List <TableDefinition>(); TableDefinition tableDefinition = new TableDefinition { Id = id.ToString(), Name = name, FieldDefinitions = new List <FieldDefinition>() }; id += 1; tableDefinition.FieldDefinitions.Add(new FieldDefinition() { Id = id.ToString(), Name = "UKPRN", IdentifierFieldType = IdentifierFieldType.UKPRN, Type = FieldType.String, Required = true }); foreach (TemplateMetadataCalculation calculation in templateContent.Calculations) { id += 1; FieldDefinition fieldDefinition = new FieldDefinition() { Id = id.ToString(), Name = calculation.Name, Required = false, Type = GetFieldType(calculation.Type), IsAggregable = calculation.AggregationType != AggregationType.None }; tableDefinition.FieldDefinitions.Add(fieldDefinition); } datasetDefinition.TableDefinitions.Add(tableDefinition); return(datasetDefinition); }
public async Task UpsertDatasetDefinitionDelegatesToTheRepository() { DatasetDefinition definition = new DatasetDefinition(); IActionResult result = await _graphService.UpsertDatasetDefinition(definition); await _datasetRepository .Received(1) .UpsertDatasetDefinition(definition); result .Should() .BeOfType <OkResult>(); }
public async Task GetDatasetSchemaSasUrl_GivenModelAndDatasetNameContainsSlashes_ReplacesSlashesWithUnderscoreAndReturnsUrl() { // Arrange DatasetSchemaSasUrlRequestModel model = new DatasetSchemaSasUrlRequestModel { DatasetDefinitionId = "12345" }; string json = JsonConvert.SerializeObject(model); byte[] byteArray = Encoding.UTF8.GetBytes(json); MemoryStream stream = new MemoryStream(byteArray); HttpRequest request = Substitute.For <HttpRequest>(); request .Body .Returns(stream); IBlobClient blobClient = CreateBlobClient(); DatasetDefinition datasetDefinition = new DatasetDefinition() { Id = "12345", Name = "TEST/SLASH Definition", }; IDatasetRepository datasetRepository = CreateDataSetsRepository(); datasetRepository .GetDatasetDefinition(Arg.Is(model.DatasetDefinitionId)) .Returns(datasetDefinition); DefinitionsService definitionsService = CreateDefinitionsService( datasetsRepository: datasetRepository, blobClient: blobClient); // Act IActionResult result = await definitionsService.GetDatasetSchemaSasUrl(request); // Assert result .Should() .BeOfType <OkObjectResult>(); blobClient .Received(1) .GetBlobSasUrl(Arg.Is("schemas/TEST_SLASH Definition.xlsx"), Arg.Any <DateTimeOffset>(), Arg.Any <SharedAccessBlobPermissions>()); }
public void OnGet_WhenSpecificationContentIsNullThenInvalidOperationExpectionIsReturned() { // Arrange IDatasetsApiClient datasetClient = Substitute.For <IDatasetsApiClient>(); ISpecsApiClient specsClient = Substitute.For <ISpecsApiClient>(); IMapper mapper = MappingHelper.CreateFrontEndMapper(); ILogger logger = Substitute.For <ILogger>(); string expectedSpecificationId = "1"; SpecificationSummary expectedSpecification = null; AssignDatasetSchemaPageModel datasetSchemaPageModel = CreatePageModel(specsClient, datasetClient, mapper); specsClient .GetSpecificationSummary(expectedSpecificationId) .Returns(new ApiResponse <SpecificationSummary>(HttpStatusCode.OK, expectedSpecification)); DatasetDefinition d1 = new DatasetDefinition() { Id = "1", Name = "APT Final Baselines", Description = "Local Authority return of provider baselines", }; DatasetDefinition d2 = new DatasetDefinition() { Id = "2", Name = "School Provider Dataset", Description = "List of providers", }; IEnumerable <DatasetDefinition> dataDefn = new List <DatasetDefinition> { d1, d2 }; datasetClient .GetDataDefinitions() .Returns(new ApiResponse <IEnumerable <DatasetDefinition> >(HttpStatusCode.OK, dataDefn)); // Act Func <Task> test = async() => await datasetSchemaPageModel.OnGet(expectedSpecificationId); // Assert test.Should().ThrowExactly <InvalidOperationException>(); }
public void Write_GivenNullTableDefinitions_ReturnsNull() { //Arrange DatasetDefinition datasetDefinition = new DatasetDefinition(); DataDefinitionExcelWriter writer = new DataDefinitionExcelWriter(); //Act byte[] excelBytes = writer.Write(datasetDefinition); //Assert excelBytes .Should() .BeNull(); }
public async Task <IActionResult> GetDatasetDefinitionById(string datasetDefinitionId) { if (string.IsNullOrWhiteSpace(datasetDefinitionId)) { _logger.Error("No datasetDefinitionId was provided to GetDatasetDefinitionById"); return(new BadRequestObjectResult("Null or empty datasetDefinitionId provided")); } DatasetDefinition defintion = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(datasetDefinitionId)); if (defintion == null) { return(new NotFoundResult()); } return(new OkObjectResult(defintion)); }
private async Task <BuildProject> ProcessDataset(Dataset dataset, string specificationId, string relationshipId, int version, Reference user) { string dataDefinitionId = dataset.Definition.Id; DatasetVersion datasetVersion = dataset.History.Where(v => v.Version == version).SingleOrDefault(); if (datasetVersion == null) { _logger.Error("Dataset version not found for dataset '{name}' ({id}) version '{version}'", dataset.Id, dataset.Name, version); throw new NonRetriableException($"Dataset version not found for dataset '{dataset.Name}' ({dataset.Name}) version '{version}'"); } string fullBlobName = datasetVersion.BlobName; DatasetDefinition datasetDefinition = (await _datasetRepository.GetDatasetDefinitionsByQuery(m => m.Id == dataDefinitionId))?.FirstOrDefault(); if (datasetDefinition == null) { _logger.Error($"Unable to find a data definition for id: {dataDefinitionId}, for blob: {fullBlobName}"); throw new NonRetriableException($"Unable to find a data definition for id: {dataDefinitionId}, for blob: {fullBlobName}"); } BuildProject buildProject = await _calcsRepository.GetBuildProjectBySpecificationId(specificationId); if (buildProject == null) { _logger.Error($"Unable to find a build project for specification id: {specificationId}"); throw new NonRetriableException($"Unable to find a build project for id: {specificationId}"); } TableLoadResult loadResult = await GetTableResult(fullBlobName, datasetDefinition); if (loadResult == null) { _logger.Error($"Failed to load table result"); throw new NonRetriableException($"Failed to load table result"); } await PersistDataset(loadResult, dataset, datasetDefinition, buildProject, specificationId, relationshipId, version, user); return(buildProject); }
private static IEnumerable <StatementSyntax> GetMembers(DatasetDefinition datasetDefinition) { IList <StatementSyntax> members = new List <StatementSyntax> { CreateStaticDefinitionName(datasetDefinition), CreateStaticDefinitionId(datasetDefinition) }; foreach (StatementSyntax member in datasetDefinition.TableDefinitions.First().FieldDefinitions.Select(GetMember)) { members.Add(member); } members.Add(GetHasValue()); return(members); }
public void Write_GivenValidDefinitionWithTwoTableDefinitions_CreatesExcelBytes() { //Arrange DatasetDefinition datasetDefinition = CreateDatasetDefinitionWithTwoTableDefinitions(); DataDefinitionExcelWriter writer = new DataDefinitionExcelWriter(); //Act byte[] excelBytes = writer.Write(datasetDefinition); //Assert excelBytes .Should() .NotBeNull(); excelBytes .Length .Should() .BeGreaterThan(0); using (Stream excelStream = new MemoryStream(excelBytes)) { ExcelPackage excelPackage = new ExcelPackage(excelStream); excelPackage.Workbook.Should().NotBeNull(); excelPackage.Workbook.Worksheets.Count.Should().Be(2); excelPackage.Workbook.Worksheets.First().Name.Should().Be("Test Table Def 1"); excelPackage.Workbook.Worksheets.First().Cells.Count().Should().Be(2); excelPackage.Workbook.Worksheets.First().Cells[1, 1].Value.Should().Be("Test field name 1"); excelPackage.Workbook.Worksheets.First().Cells[1, 1].Comment.Should().NotBeNull(); excelPackage.Workbook.Worksheets.First().Cells[1, 1].Comment.Text.Should().Be("Description: Test description 1\n Required: No\n Type: String"); excelPackage.Workbook.Worksheets.First().Cells[1, 2].Value.Should().Be("Test field name 2"); excelPackage.Workbook.Worksheets.First().Cells[1, 2].Comment.Should().NotBeNull(); excelPackage.Workbook.Worksheets.First().Cells[1, 2].Comment.Text.Should().Be("Description: Test description 2\n Required: No\n Type: String"); excelPackage.Workbook.Worksheets.Last().Name.Should().Be("Test Table Def 2"); excelPackage.Workbook.Worksheets.Last().Cells.Count().Should().Be(2); excelPackage.Workbook.Worksheets.Last().Cells[1, 1].Value.Should().Be("Test field name 3"); excelPackage.Workbook.Worksheets.Last().Cells[1, 1].Comment.Should().NotBeNull(); excelPackage.Workbook.Worksheets.Last().Cells[1, 1].Comment.Text.Should().Be("Description: Test description 3\n Required: No\n Type: String"); excelPackage.Workbook.Worksheets.Last().Cells[1, 2].Value.Should().Be("Test field name 4"); excelPackage.Workbook.Worksheets.Last().Cells[1, 2].Comment.Should().NotBeNull(); excelPackage.Workbook.Worksheets.Last().Cells[1, 2].Comment.Text.Should().Be("Description: Test description 4\n Required: No\n Type: String"); } }
private static StatementSyntax CreateStaticDefinitionId(DatasetDefinition datasetDefinition) { SyntaxToken token = SyntaxFactory.Literal(datasetDefinition.Id); VariableDeclaratorSyntax variable = SyntaxFactory.VariableDeclarator( SyntaxFactory.SingletonSeparatedList(SyntaxFactory.ModifiedIdentifier("DatasetDefinitionId"))); variable = variable.WithAsClause( SyntaxFactory.SimpleAsClause(SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.StringKeyword)))); variable = variable.WithInitializer( SyntaxFactory.EqualsValue(SyntaxFactory.LiteralExpression(SyntaxKind.StringLiteralExpression, token))); return(SyntaxFactory.FieldDeclaration( SyntaxFactory.List <AttributeListSyntax>(), SyntaxFactory.TokenList(SyntaxFactory.Token(SyntaxKind.PublicKeyword), SyntaxFactory.Token(SyntaxKind.SharedKeyword)), SyntaxFactory.SingletonSeparatedList(variable))); }