async public Task GetDatasetDefinitions_GivenDefinitionsRequestedButContainsNoResults_ReturnsEmptyArray()
{
//Arrange
HttpRequest request = Substitute.For <HttpRequest>();
IEnumerable <DatasetDefinition> definitions = new DatasetDefinition[0];
IDatasetRepository repository = CreateDataSetsRepository();
repository
.GetDatasetDefinitions()
.Returns(definitions);
DefinitionsService service = CreateDefinitionsService(datasetsRepository: repository);
//Act
IActionResult result = await service.GetDatasetDefinitions(request);
//Assert
result
.Should()
.BeOfType <OkObjectResult>();
OkObjectResult objResult = (OkObjectResult)result;
IEnumerable <DatasetDefinition> objValue = (IEnumerable <DatasetDefinition>)objResult.Value;
objValue
.Count()
.Should()
.Be(0);
}
public async Task <IActionResult> GetDatasetSchemaSasUrl(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
DatasetSchemaSasUrlRequestModel requestModel = JsonConvert.DeserializeObject <DatasetSchemaSasUrlRequestModel>(json);
if (requestModel == null)
{
_logger.Warning("No dataset schema request model was provided");
return(new BadRequestObjectResult("No dataset schema request model was provided"));
}
if (requestModel.DatasetDefinitionId.IsNullOrEmpty())
{
_logger.Warning("No dataset schema name was provided");
return(new BadRequestObjectResult("No dataset schema name was provided"));
}
DatasetDefinition datasetDefinition = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(requestModel.DatasetDefinitionId));
if (datasetDefinition == null)
{
return(new NotFoundObjectResult("Data schema definiton not found"));
}
string definitionName = datasetDefinition.Name.Replace("/", "_").Replace("\\", "_");
string fileName = $"schemas/{definitionName}.xlsx";
string blobUrl = _blobClient.GetBlobSasUrl(fileName, DateTimeOffset.UtcNow.AddDays(1), SharedAccessBlobPermissions.Read);
return(new OkObjectResult(new DatasetSchemaSasUrlResponseModel {
SchemaUrl = blobUrl
}));
}
public DatasetUploadValidationModel(ExcelPackage excelPackage, Func <IEnumerable <ProviderSummary> > providerSummaries, DatasetDefinition datasetDefinition)
{
ExcelPackage = excelPackage;
ProviderSummaries = providerSummaries;
DatasetDefinition = datasetDefinition;
ValidationResult = new DatasetUploadValidationResult();
}
public void Validate_GivenDuplicateDatasetDefinitionName_ValidIsFalse()
{
//Arrange
DatasetDefinition model = CreateModel();
model.Name = DatasetDefinitionName;
model.Id = DatasetDefinitionId;
DatasetDefinitionValidator validator = CreateValidator();
//Act
ValidationResult result = validator.Validate(model);
//Assert
result
.IsValid
.Should()
.BeFalse();
result
.Errors
.Count
.Should()
.Be(1);
}
private void AssertDatasetIsPopulated(DatasetDefinition dataset)
{
Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.DatasetName));
Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.DisplayName));
Assert.IsNotEmpty(dataset.IndexBy);
if (UseMocks)
{
Assert.IsNotNull(dataset.LastScheduledDate);
Assert.IsNotNull(dataset.LatestSubtimeframeAvailable);
}
Assert.IsNotNull(dataset.Query);
Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.ProjectId));
Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.AnalysisType));
Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.EventCollection));
Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.Timeframe));
Assert.IsTrue(!string.IsNullOrWhiteSpace(dataset.Query.Interval));
// TODO : We'll need to do some setup to actually get this to run automatically
// with !UseMocks...and take into account that it can "take up to an hour for a newly
// created dataset to compute results for the first time."
Assert.IsNotNull(dataset.Query.GroupBy);
Assert.IsTrue(dataset.Query.GroupBy.Count() == 1);
if (dataset.Query.Filters != null)
{
foreach (var filter in dataset.Query.Filters)
{
AssertFilterIsPopulated(filter);
}
}
}
public DatasetDefinitionChanges DetectChanges(DatasetDefinition newDatasetDefinition, DatasetDefinition existingDatasetDefinition)
{
Guard.ArgumentNotNull(newDatasetDefinition, nameof(newDatasetDefinition));
Guard.ArgumentNotNull(existingDatasetDefinition, nameof(existingDatasetDefinition));
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges
{
Id = newDatasetDefinition.Id
};
string newDatasetDefinitionAsJson = JsonConvert.SerializeObject(newDatasetDefinition);
string existingDatasetDefinitionAsJson = JsonConvert.SerializeObject(existingDatasetDefinition);
if (string.Equals(newDatasetDefinitionAsJson, existingDatasetDefinitionAsJson))
{
return(datasetDefinitionChanges);
}
if (newDatasetDefinition.Name != existingDatasetDefinition.Name)
{
datasetDefinitionChanges.DefinitionChanges.Add(DefinitionChangeType.DefinitionName);
datasetDefinitionChanges.NewName = newDatasetDefinition.Name;
}
datasetDefinitionChanges.TableDefinitionChanges.AddRange(DetectTableDefinitionChanges(existingDatasetDefinition.TableDefinitions, newDatasetDefinition.TableDefinitions));
return(datasetDefinitionChanges);
}
public async Task <HttpStatusCode> UpsertDatasetDefinition(DatasetDefinition definition)
{
Guard.ArgumentNotNull(definition, nameof(definition));
return(await PostAsync($"{UrlRoot}/datasetdefinition",
definition));
}
private async Task <TableLoadResult> GetTableResult(string fullBlobName, DatasetDefinition datasetDefinition)
{
string dataset_cache_key = $"{CacheKeys.DatasetRows}:{datasetDefinition.Id}:{GetBlobNameCacheKey(fullBlobName)}".ToLowerInvariant();
IEnumerable <TableLoadResult> tableLoadResults = await _cacheProvider.GetAsync <TableLoadResult[]>(dataset_cache_key);
if (tableLoadResults.IsNullOrEmpty())
{
ICloudBlob blob = await _blobClient.GetBlobReferenceFromServerAsync(fullBlobName);
if (blob == null)
{
_logger.Error($"Failed to find blob with path: {fullBlobName}");
throw new NonRetriableException($"Failed to find blob with path: {fullBlobName}");
}
using (Stream datasetStream = await _blobClient.DownloadToStreamAsync(blob))
{
if (datasetStream == null || datasetStream.Length == 0)
{
_logger.Error($"Invalid blob returned: {fullBlobName}");
throw new NonRetriableException($"Invalid blob returned: {fullBlobName}");
}
tableLoadResults = _excelDatasetReader.Read(datasetStream, datasetDefinition).ToList();
}
await _cacheProvider.SetAsync(dataset_cache_key, tableLoadResults.ToArraySafe(), TimeSpan.FromDays(7), true);
}
return(tableLoadResults.FirstOrDefault());
}
public async Task <IActionResult> CreateOrUpdateDatasetDefinition(CreateDatasetDefinitionFromTemplateModel model, string correlationId, Reference user)
{
ValidationResult validationResult = await _createDatasetDefinitionFromTemplateValidator.ValidateAsync(model);
if (!validationResult.IsValid)
{
string errorMessage = string.Join(";", validationResult.Errors.Select(x => x.ErrorMessage));
_logger.Error(errorMessage);
return(new BadRequestObjectResult(errorMessage));
}
;
FundingStream fundingStream = await _policyRepository.GetFundingStream(model.FundingStreamId);
TemplateMetadataDistinctCalculationsContents templateContents = await _policyRepository.GetDistinctTemplateMetadataCalculationsContents(model.FundingStreamId, model.FundingPeriodId, model.TemplateVersion);
if (templateContents == null)
{
return(new BadRequestObjectResult($"No funding template for given FundingStreamId " +
$"- {model.FundingStreamId}, FundingPeriodId - {model.FundingPeriodId}, TemplateVersion - {model.TemplateVersion}"));
}
DatasetDefinition datasetDefinition = CreateDatasetDefinition(model, templateContents, fundingStream);
return(await SaveDatasetDefinition(datasetDefinition, correlationId, user));
}
private static IEnumerable <string> GetProviderIdsForIdentifier(DatasetDefinition datasetDefinition, RowLoadResult row, IEnumerable <ProviderSummary> providerSummaries)
{
IEnumerable <FieldDefinition> identifierFields = datasetDefinition.TableDefinitions?.First().FieldDefinitions.Where(x => x.IdentifierFieldType.HasValue);
foreach (FieldDefinition field in identifierFields)
{
if (!string.IsNullOrWhiteSpace(field.Name))
{
if (row.Fields.ContainsKey(field.Name))
{
string identifier = row.Fields[field.Name]?.ToString();
if (!string.IsNullOrWhiteSpace(identifier))
{
Dictionary <string, List <string> > lookup = GetDictionaryForIdentifierType(field.IdentifierFieldType, identifier, providerSummaries);
if (lookup.TryGetValue(identifier, out List <string> providerIds))
{
return(providerIds);
}
}
else
{
// For debugging only
//_logger.Debug("Found identifier with null or emtpy string for provider");
}
}
}
}
return(new string[0]);
}
public void DetectChanges_GivenTableDefinitionNameChange_ReturnsModelWithChanges()
{
//Arrange
DatasetDefinition newDefinition = new DatasetDefinition
{
Name = "name2",
TableDefinitions = new List <TableDefinition>
{
new TableDefinition {
Name = "name2", FieldDefinitions = new List <FieldDefinition>()
}
}
};
DatasetDefinition existingDefinition = new DatasetDefinition
{
Name = "name2",
TableDefinitions = new List <TableDefinition>
{
new TableDefinition {
Name = "name1", FieldDefinitions = new List <FieldDefinition>()
}
}
};
DefinitionChangesDetectionService service = new DefinitionChangesDetectionService();
//Act
DatasetDefinitionChanges changes = service.DetectChanges(newDefinition, existingDefinition);
//Assert
changes
.HasChanges
.Should()
.BeTrue();
changes
.TableDefinitionChanges
.First()
.HasChanges
.Should()
.BeTrue();
changes
.TableDefinitionChanges
.First()
.ChangeTypes
.First()
.Should()
.Be(TableDefinitionChangeType.DefinitionName);
changes
.TableDefinitionChanges
.First()
.TableDefinition
.Name
.Should()
.Be("name2");
}
public async Task UpsertDatasetDefinitionDelegatesToGraphRepository()
{
DatasetDefinition definition = NewDataDefinition();
await _datasetRepository.UpsertDatasetDefinition(definition);
await ThenTheNodeWasCreated(definition, AttributeConstants.DatasetDefinitionId);
}
private static bool UploadDataset(string subscriptionKey, string hostURI, DatasetDefinition datasetDefinition, string wavePath, string scriptPath)
{
string waveName = Path.GetFileName(wavePath);
string scriptName = Path.GetFileName(scriptPath);
using (FileStream fsscript = new FileStream(scriptPath, FileMode.Open))
using (FileStream fswave = new FileStream(wavePath, FileMode.Open))
using (var client = new HttpClient())
using (var content = new MultipartFormDataContent())
{
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey);
content.Add(new StringContent(datasetDefinition.Name), "name");
if (datasetDefinition.Description != null)
{
content.Add(new StringContent(datasetDefinition.Description), "description");
}
if (datasetDefinition.ProjectId != null)
{
content.Add(new StringContent(datasetDefinition.ProjectId), "projectId");
}
content.Add(new StringContent(datasetDefinition.DataImportKind), "dataImportKind");
content.Add(new StringContent(datasetDefinition.Locale), "locale");
if (datasetDefinition.Properties != null)
{
content.Add(new StringContent(JsonConvert.SerializeObject(datasetDefinition.Properties)), "properties");
}
var transcriptionContent = new StreamContent(fsscript);
transcriptionContent.Headers.Add("Content-Disposition", $@"form-data; name=""transcriptions""; filename=""{scriptName}""");
transcriptionContent.Headers.Add("Content-Type", "text/plain");
transcriptionContent.Headers.Add("Content-Length", $"{fsscript.Length}");
content.Add(transcriptionContent, "transcriptions", scriptName);
var wavesContent = new StreamContent(fswave);
wavesContent.Headers.Add("Content-Disposition", $@"form-data; name=""audiodata""; filename=""{waveName}""");
wavesContent.Headers.Add("Content-Type", "application/x-zip-compressed");
wavesContent.Headers.Add("Content-Length", $"{fswave.Length}");
content.Add(wavesContent, "audiodata", waveName);
string url = string.Format(CultureInfo.InvariantCulture, hostURI + API_V3.VoiceDatasets_Upload);
var response = client.PostAsync(url, content).Result;
if (response.StatusCode != HttpStatusCode.Accepted)
{
APIHelper.PrintErrorMessage(response);
return(false);
}
System.Console.WriteLine(response.Headers.Location);
return(true);
}
}
//Create Long Audio Dataset
public void UploadLongAudioDataset(string waveZipUpload, string scriptZipUpload, string name, string description, string locale, string gender)
{
var properties = new Dictionary <string, string>
{
{ "Gender", gender }
};
var datasetDefinition = new DatasetDefinition(name, description, locale, properties, "CustomVoice");
var submitResponse = VoiceAPIHelper.SubmitLongAudioDataset(datasetDefinition, waveZipUpload, scriptZipUpload, CreateLongAudioDatasetUrl, this.subscriptionKey);
}
public TableLoadResultWithHeaders Read(ExcelPackage excelPackage, DatasetDefinition datasetDefinition, bool parse)
{
if (datasetDefinition.TableDefinitions.Count == 1 && excelPackage.Workbook.Worksheets.Count > 0)
{
return(ConvertSheetToObjects(excelPackage.Workbook.Worksheets.First(), datasetDefinition.TableDefinitions.First(), parse));
}
return(null);
}
public async Task Merge_ShouldGetTheNewAndUpdatedData(string latestBlobFileName, string blobFileNameToMerge, string resultsFile, string definitionFileName)
{
DatasetDefinition datasetDefinition = GetDatasetDefinitionByName(definitionFileName);
await using Stream latestDatasetStream = File.OpenRead($"TestItems{Path.DirectorySeparatorChar}{latestBlobFileName}");
Mock <ICloudBlob> latestFileBlob = new Mock <ICloudBlob>();
_blobClient.Setup(_ => _
.GetBlobReferenceFromServerAsync(latestBlobFileName))
.ReturnsAsync(latestFileBlob.Object);
_blobClient.Setup(_ => _
.DownloadToStreamAsync(latestFileBlob.Object))
.ReturnsAsync(latestDatasetStream);
await using Stream fileToMergeDatasetStream = File.OpenRead($"TestItems{Path.DirectorySeparatorChar}{blobFileNameToMerge}");
Mock <ICloudBlob> fileToMergeBlob = new Mock <ICloudBlob>();
_blobClient.Setup(_ => _
.GetBlobReferenceFromServerAsync(blobFileNameToMerge))
.ReturnsAsync(fileToMergeBlob.Object);
_blobClient.Setup(_ => _
.DownloadToStreamAsync(fileToMergeBlob.Object))
.ReturnsAsync(fileToMergeDatasetStream);
await using MemoryStream uploadedStream = new MemoryStream();
fileToMergeBlob.Setup(_ => _.UploadFromStreamAsync(It.IsAny <Stream>()))
.Callback <Stream>(_ =>
{
_?.Seek(0, SeekOrigin.Begin);
// ReSharper disable once AccessToDisposedClosure
_?.CopyTo(uploadedStream);
});
DatasetDataMergeResult result = await _service.Merge(datasetDefinition, latestBlobFileName, blobFileNameToMerge);
result.TablesMergeResults.Count().Should().Be(1);
await using Stream expectedResultStream = File.OpenRead($"TestItems{Path.DirectorySeparatorChar}{resultsFile}");
using ExcelPackage expected = new ExcelPackage(expectedResultStream);
using ExcelPackage actual = new ExcelPackage(uploadedStream);
ExcelWorksheet expectedWorksheet = expected.Workbook.Worksheets[1];
ExcelWorksheet actualWorksheet = expected.Workbook.Worksheets[1];
for (int i = 1; i <= expectedWorksheet.Dimension.Rows; i++)
{
for (int j = 1; j <= expectedWorksheet.Dimension.Columns; j++)
{
actualWorksheet.Cells[i, j].Value.Should().Be(expectedWorksheet.Cells[i, j].Value);
}
}
}
private async Task <IEnumerable <IndexError> > IndexDatasetDefinition(DatasetDefinition definition, PoliciesApiModels.FundingStream fundingStream)
{
// Calculate hash for model to see if there are changes
string modelJson = JsonConvert.SerializeObject(definition);
string hashCode = "";
using (SHA256 sha256Generator = SHA256Managed.Create())
{
byte[] modelBytes = UTF8Encoding.UTF8.GetBytes(modelJson);
foreach (byte hashByte in sha256Generator.ComputeHash(modelBytes))
{
hashCode += string.Format("{0:X2}", hashByte);
}
}
DatasetDefinitionIndex datasetDefinitionIndex = new DatasetDefinitionIndex()
{
Id = definition.Id,
Name = definition.Name,
Description = definition.Description,
LastUpdatedDate = DateTimeOffset.Now,
ProviderIdentifier = definition.TableDefinitions.FirstOrDefault()?.FieldDefinitions?.Where(f => f.IdentifierFieldType.HasValue)?.Select(f => Enum.GetName(typeof(IdentifierFieldType), f.IdentifierFieldType.Value)).FirstOrDefault(),
ModelHash = hashCode,
FundingStreamId = fundingStream.Id,
FundingStreamName = fundingStream.Name
};
if (string.IsNullOrWhiteSpace(datasetDefinitionIndex.ProviderIdentifier))
{
datasetDefinitionIndex.ProviderIdentifier = "None";
}
bool updateIndex = true;
// Only update index if metadata or model has changed, this is to preserve the LastUpdateDate
DatasetDefinitionIndex existingIndex = await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() =>
_datasetDefinitionSearchRepository.SearchById(definition.Id));
if (existingIndex != null)
{
if (existingIndex.ModelHash == hashCode &&
existingIndex.Description == definition.Description &&
existingIndex.Name == definition.Name &&
existingIndex.ProviderIdentifier == datasetDefinitionIndex.ProviderIdentifier)
{
updateIndex = false;
}
}
if (updateIndex)
{
return(await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() =>
_datasetDefinitionSearchRepository.Index(new DatasetDefinitionIndex[] { datasetDefinitionIndex })));
}
return(Enumerable.Empty <IndexError>());
}
private static DatasetDefinition GetDatasetDefinitionByName(string datasetDefinitionName)
{
JObject obj1 =
JObject.Parse(File.ReadAllText($"DatasetDefinitions{Path.DirectorySeparatorChar}{datasetDefinitionName}"));
DatasetDefinition datasetDefinition = obj1.ToObject <DatasetDefinition>();
return(datasetDefinition);
}
private async Task <BuildProject> GenerateBuildProject(string specificationId)
{
BuildProject buildproject = new BuildProject
{
SpecificationId = specificationId,
Id = Guid.NewGuid().ToString(),
Name = specificationId,
DatasetRelationships = new List <DatasetRelationshipSummary>(),
Build = new Build()
};
IEnumerable <DatasetSpecificationRelationshipViewModel> datasetRelationshipModels = await _datasetRepositoryPolicy.ExecuteAsync(() => _datasetRepository.GetCurrentRelationshipsBySpecificationId(specificationId));
if (!datasetRelationshipModels.IsNullOrEmpty())
{
ConcurrentBag <DatasetDefinition> datasetDefinitions = new ConcurrentBag <DatasetDefinition>();
IList <Task> definitionTasks = new List <Task>();
IEnumerable <string> definitionIds = datasetRelationshipModels.Select(m => m.Definition?.Id);
foreach (string definitionId in definitionIds)
{
Task task = Task.Run(async() =>
{
DatasetDefinition datasetDefinition = await _datasetRepositoryPolicy.ExecuteAsync(() => _datasetRepository.GetDatasetDefinitionById(definitionId));
if (datasetDefinition != null)
{
datasetDefinitions.Add(datasetDefinition);
}
});
definitionTasks.Add(task);
}
await TaskHelper.WhenAllAndThrow(definitionTasks.ToArray());
foreach (DatasetSpecificationRelationshipViewModel datasetRelationshipModel in datasetRelationshipModels)
{
buildproject.DatasetRelationships.Add(new DatasetRelationshipSummary
{
DatasetDefinitionId = datasetRelationshipModel.Definition.Id,
DatasetId = datasetRelationshipModel.DatasetId,
Relationship = new Common.Models.Reference(datasetRelationshipModel.Id, datasetRelationshipModel.Name),
DefinesScope = datasetRelationshipModel.IsProviderData,
Id = datasetRelationshipModel.Id,
Name = datasetRelationshipModel.Name,
DatasetDefinition = datasetDefinitions.FirstOrDefault(m => m.Id == datasetRelationshipModel.Definition.Id)
});
}
}
return(buildproject);
}
public async Task OnGet_WhenSpecificationStatusCodeIsNotFoundThenNotFoundObjectResultReturned()
{
// Arrange
IDatasetsApiClient datasetClient = Substitute.For <IDatasetsApiClient>();
ISpecsApiClient specsClient = Substitute.For <ISpecsApiClient>();
IMapper mapper = MappingHelper.CreateFrontEndMapper();
ILogger logger = Substitute.For <ILogger>();
string expectedSpecificationId = "1";
Specification expectedSpecification = null;
AssignDatasetSchemaPageModel datasetSchemaPageModel = CreatePageModel(specsClient, datasetClient, mapper);
specsClient
.GetSpecification(expectedSpecificationId)
.Returns(new ApiResponse <Specification>(HttpStatusCode.NotFound, expectedSpecification));
DatasetDefinition d1 = new DatasetDefinition()
{
Id = "1",
Name = "APT Final Baselines",
Description = "Local Authority return of provider baselines",
};
DatasetDefinition d2 = new DatasetDefinition()
{
Id = "2",
Name = "School Provider Dataset",
Description = "List of providers",
};
IEnumerable <DatasetDefinition> dataDefn = new List <DatasetDefinition> {
d1, d2
};
datasetClient
.GetDataDefinitions()
.Returns(new ApiResponse <IEnumerable <DatasetDefinition> >(HttpStatusCode.OK, dataDefn));
// Act
IActionResult result = await datasetSchemaPageModel.OnGet(expectedSpecificationId);
// Assert
result.Should().NotBeNull();
result.Should().BeOfType <NotFoundObjectResult>();
NotFoundObjectResult typeResult = result as NotFoundObjectResult;
typeResult.Value.Should().Be("Unable to get specification response. Specification Id value = 1");
Assert.IsInstanceOfType(result, typeof(NotFoundObjectResult));
}
private static DatasetDefinition CreateDatasetDefinition(
CreateDatasetDefinitionFromTemplateModel model,
TemplateMetadataDistinctCalculationsContents templateContent,
FundingStream fundingStream)
{
int id = model.DatasetDefinitionId;
string name = $"{fundingStream.Name}-{model.TemplateVersion}";
DatasetDefinition datasetDefinition = new DatasetDefinition()
{
Id = id.ToString(),
Name = name,
Description = name,
FundingStreamId = fundingStream.Id
};
id += 1;
datasetDefinition.TableDefinitions = new List <TableDefinition>();
TableDefinition tableDefinition = new TableDefinition
{
Id = id.ToString(),
Name = name,
FieldDefinitions = new List <FieldDefinition>()
};
id += 1;
tableDefinition.FieldDefinitions.Add(new FieldDefinition()
{
Id = id.ToString(),
Name = "UKPRN",
IdentifierFieldType = IdentifierFieldType.UKPRN,
Type = FieldType.String,
Required = true
});
foreach (TemplateMetadataCalculation calculation in templateContent.Calculations)
{
id += 1;
FieldDefinition fieldDefinition = new FieldDefinition()
{
Id = id.ToString(),
Name = calculation.Name,
Required = false,
Type = GetFieldType(calculation.Type),
IsAggregable = calculation.AggregationType != AggregationType.None
};
tableDefinition.FieldDefinitions.Add(fieldDefinition);
}
datasetDefinition.TableDefinitions.Add(tableDefinition);
return(datasetDefinition);
}
public async Task UpsertDatasetDefinitionDelegatesToTheRepository()
{
DatasetDefinition definition = new DatasetDefinition();
IActionResult result = await _graphService.UpsertDatasetDefinition(definition);
await _datasetRepository
.Received(1)
.UpsertDatasetDefinition(definition);
result
.Should()
.BeOfType <OkResult>();
}
public async Task GetDatasetSchemaSasUrl_GivenModelAndDatasetNameContainsSlashes_ReplacesSlashesWithUnderscoreAndReturnsUrl()
{
// Arrange
DatasetSchemaSasUrlRequestModel model = new DatasetSchemaSasUrlRequestModel
{
DatasetDefinitionId = "12345"
};
string json = JsonConvert.SerializeObject(model);
byte[] byteArray = Encoding.UTF8.GetBytes(json);
MemoryStream stream = new MemoryStream(byteArray);
HttpRequest request = Substitute.For <HttpRequest>();
request
.Body
.Returns(stream);
IBlobClient blobClient = CreateBlobClient();
DatasetDefinition datasetDefinition = new DatasetDefinition()
{
Id = "12345",
Name = "TEST/SLASH Definition",
};
IDatasetRepository datasetRepository = CreateDataSetsRepository();
datasetRepository
.GetDatasetDefinition(Arg.Is(model.DatasetDefinitionId))
.Returns(datasetDefinition);
DefinitionsService definitionsService = CreateDefinitionsService(
datasetsRepository: datasetRepository,
blobClient: blobClient);
// Act
IActionResult result = await definitionsService.GetDatasetSchemaSasUrl(request);
// Assert
result
.Should()
.BeOfType <OkObjectResult>();
blobClient
.Received(1)
.GetBlobSasUrl(Arg.Is("schemas/TEST_SLASH Definition.xlsx"), Arg.Any <DateTimeOffset>(), Arg.Any <SharedAccessBlobPermissions>());
}
public void OnGet_WhenSpecificationContentIsNullThenInvalidOperationExpectionIsReturned()
{
// Arrange
IDatasetsApiClient datasetClient = Substitute.For <IDatasetsApiClient>();
ISpecsApiClient specsClient = Substitute.For <ISpecsApiClient>();
IMapper mapper = MappingHelper.CreateFrontEndMapper();
ILogger logger = Substitute.For <ILogger>();
string expectedSpecificationId = "1";
SpecificationSummary expectedSpecification = null;
AssignDatasetSchemaPageModel datasetSchemaPageModel = CreatePageModel(specsClient, datasetClient, mapper);
specsClient
.GetSpecificationSummary(expectedSpecificationId)
.Returns(new ApiResponse <SpecificationSummary>(HttpStatusCode.OK, expectedSpecification));
DatasetDefinition d1 = new DatasetDefinition()
{
Id = "1",
Name = "APT Final Baselines",
Description = "Local Authority return of provider baselines",
};
DatasetDefinition d2 = new DatasetDefinition()
{
Id = "2",
Name = "School Provider Dataset",
Description = "List of providers",
};
IEnumerable <DatasetDefinition> dataDefn = new List <DatasetDefinition> {
d1, d2
};
datasetClient
.GetDataDefinitions()
.Returns(new ApiResponse <IEnumerable <DatasetDefinition> >(HttpStatusCode.OK, dataDefn));
// Act
Func <Task> test = async() => await datasetSchemaPageModel.OnGet(expectedSpecificationId);
// Assert
test.Should().ThrowExactly <InvalidOperationException>();
}
public void Write_GivenNullTableDefinitions_ReturnsNull()
{
//Arrange
DatasetDefinition datasetDefinition = new DatasetDefinition();
DataDefinitionExcelWriter writer = new DataDefinitionExcelWriter();
//Act
byte[] excelBytes = writer.Write(datasetDefinition);
//Assert
excelBytes
.Should()
.BeNull();
}
public async Task <IActionResult> GetDatasetDefinitionById(string datasetDefinitionId)
{
if (string.IsNullOrWhiteSpace(datasetDefinitionId))
{
_logger.Error("No datasetDefinitionId was provided to GetDatasetDefinitionById");
return(new BadRequestObjectResult("Null or empty datasetDefinitionId provided"));
}
DatasetDefinition defintion = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(datasetDefinitionId));
if (defintion == null)
{
return(new NotFoundResult());
}
return(new OkObjectResult(defintion));
}
private async Task <BuildProject> ProcessDataset(Dataset dataset, string specificationId, string relationshipId, int version, Reference user)
{
string dataDefinitionId = dataset.Definition.Id;
DatasetVersion datasetVersion = dataset.History.Where(v => v.Version == version).SingleOrDefault();
if (datasetVersion == null)
{
_logger.Error("Dataset version not found for dataset '{name}' ({id}) version '{version}'", dataset.Id, dataset.Name, version);
throw new NonRetriableException($"Dataset version not found for dataset '{dataset.Name}' ({dataset.Name}) version '{version}'");
}
string fullBlobName = datasetVersion.BlobName;
DatasetDefinition datasetDefinition =
(await _datasetRepository.GetDatasetDefinitionsByQuery(m => m.Id == dataDefinitionId))?.FirstOrDefault();
if (datasetDefinition == null)
{
_logger.Error($"Unable to find a data definition for id: {dataDefinitionId}, for blob: {fullBlobName}");
throw new NonRetriableException($"Unable to find a data definition for id: {dataDefinitionId}, for blob: {fullBlobName}");
}
BuildProject buildProject = await _calcsRepository.GetBuildProjectBySpecificationId(specificationId);
if (buildProject == null)
{
_logger.Error($"Unable to find a build project for specification id: {specificationId}");
throw new NonRetriableException($"Unable to find a build project for id: {specificationId}");
}
TableLoadResult loadResult = await GetTableResult(fullBlobName, datasetDefinition);
if (loadResult == null)
{
_logger.Error($"Failed to load table result");
throw new NonRetriableException($"Failed to load table result");
}
await PersistDataset(loadResult, dataset, datasetDefinition, buildProject, specificationId, relationshipId, version, user);
return(buildProject);
}
private static IEnumerable <StatementSyntax> GetMembers(DatasetDefinition datasetDefinition)
{
IList <StatementSyntax> members = new List <StatementSyntax>
{
CreateStaticDefinitionName(datasetDefinition),
CreateStaticDefinitionId(datasetDefinition)
};
foreach (StatementSyntax member in datasetDefinition.TableDefinitions.First().FieldDefinitions.Select(GetMember))
{
members.Add(member);
}
members.Add(GetHasValue());
return(members);
}
public void Write_GivenValidDefinitionWithTwoTableDefinitions_CreatesExcelBytes()
{
//Arrange
DatasetDefinition datasetDefinition = CreateDatasetDefinitionWithTwoTableDefinitions();
DataDefinitionExcelWriter writer = new DataDefinitionExcelWriter();
//Act
byte[] excelBytes = writer.Write(datasetDefinition);
//Assert
excelBytes
.Should()
.NotBeNull();
excelBytes
.Length
.Should()
.BeGreaterThan(0);
using (Stream excelStream = new MemoryStream(excelBytes))
{
ExcelPackage excelPackage = new ExcelPackage(excelStream);
excelPackage.Workbook.Should().NotBeNull();
excelPackage.Workbook.Worksheets.Count.Should().Be(2);
excelPackage.Workbook.Worksheets.First().Name.Should().Be("Test Table Def 1");
excelPackage.Workbook.Worksheets.First().Cells.Count().Should().Be(2);
excelPackage.Workbook.Worksheets.First().Cells[1, 1].Value.Should().Be("Test field name 1");
excelPackage.Workbook.Worksheets.First().Cells[1, 1].Comment.Should().NotBeNull();
excelPackage.Workbook.Worksheets.First().Cells[1, 1].Comment.Text.Should().Be("Description: Test description 1\n Required: No\n Type: String");
excelPackage.Workbook.Worksheets.First().Cells[1, 2].Value.Should().Be("Test field name 2");
excelPackage.Workbook.Worksheets.First().Cells[1, 2].Comment.Should().NotBeNull();
excelPackage.Workbook.Worksheets.First().Cells[1, 2].Comment.Text.Should().Be("Description: Test description 2\n Required: No\n Type: String");
excelPackage.Workbook.Worksheets.Last().Name.Should().Be("Test Table Def 2");
excelPackage.Workbook.Worksheets.Last().Cells.Count().Should().Be(2);
excelPackage.Workbook.Worksheets.Last().Cells[1, 1].Value.Should().Be("Test field name 3");
excelPackage.Workbook.Worksheets.Last().Cells[1, 1].Comment.Should().NotBeNull();
excelPackage.Workbook.Worksheets.Last().Cells[1, 1].Comment.Text.Should().Be("Description: Test description 3\n Required: No\n Type: String");
excelPackage.Workbook.Worksheets.Last().Cells[1, 2].Value.Should().Be("Test field name 4");
excelPackage.Workbook.Worksheets.Last().Cells[1, 2].Comment.Should().NotBeNull();
excelPackage.Workbook.Worksheets.Last().Cells[1, 2].Comment.Text.Should().Be("Description: Test description 4\n Required: No\n Type: String");
}
}
private static StatementSyntax CreateStaticDefinitionId(DatasetDefinition datasetDefinition)
{
SyntaxToken token = SyntaxFactory.Literal(datasetDefinition.Id);
VariableDeclaratorSyntax variable = SyntaxFactory.VariableDeclarator(
SyntaxFactory.SingletonSeparatedList(SyntaxFactory.ModifiedIdentifier("DatasetDefinitionId")));
variable = variable.WithAsClause(
SyntaxFactory.SimpleAsClause(SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.StringKeyword))));
variable = variable.WithInitializer(
SyntaxFactory.EqualsValue(SyntaxFactory.LiteralExpression(SyntaxKind.StringLiteralExpression,
token)));
return(SyntaxFactory.FieldDeclaration(
SyntaxFactory.List <AttributeListSyntax>(),
SyntaxFactory.TokenList(SyntaxFactory.Token(SyntaxKind.PublicKeyword),
SyntaxFactory.Token(SyntaxKind.SharedKeyword)),
SyntaxFactory.SingletonSeparatedList(variable)));
}
