private async Task <IEnumerable <IndexError> > IndexDatasetDefinition(DatasetDefinition definition, PoliciesApiModels.FundingStream fundingStream)
{
// Calculate hash for model to see if there are changes
string modelJson = JsonConvert.SerializeObject(definition);
string hashCode = "";
using (SHA256 sha256Generator = SHA256Managed.Create())
{
byte[] modelBytes = UTF8Encoding.UTF8.GetBytes(modelJson);
foreach (byte hashByte in sha256Generator.ComputeHash(modelBytes))
{
hashCode += string.Format("{0:X2}", hashByte);
}
}
DatasetDefinitionIndex datasetDefinitionIndex = new DatasetDefinitionIndex()
{
Id = definition.Id,
Name = definition.Name,
Description = definition.Description,
LastUpdatedDate = DateTimeOffset.Now,
ProviderIdentifier = definition.TableDefinitions.FirstOrDefault()?.FieldDefinitions?.Where(f => f.IdentifierFieldType.HasValue)?.Select(f => Enum.GetName(typeof(IdentifierFieldType), f.IdentifierFieldType.Value)).FirstOrDefault(),
ModelHash = hashCode,
FundingStreamId = fundingStream.Id,
FundingStreamName = fundingStream.Name
};
if (string.IsNullOrWhiteSpace(datasetDefinitionIndex.ProviderIdentifier))
{
datasetDefinitionIndex.ProviderIdentifier = "None";
}
bool updateIndex = true;
// Only update index if metadata or model has changed, this is to preserve the LastUpdateDate
DatasetDefinitionIndex existingIndex = await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() =>
_datasetDefinitionSearchRepository.SearchById(definition.Id));
if (existingIndex != null)
{
if (existingIndex.ModelHash == hashCode &&
existingIndex.Description == definition.Description &&
existingIndex.Name == definition.Name &&
existingIndex.ProviderIdentifier == datasetDefinitionIndex.ProviderIdentifier)
{
updateIndex = false;
}
}
if (updateIndex)
{
return(await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() =>
_datasetDefinitionSearchRepository.Index(new DatasetDefinitionIndex[] { datasetDefinitionIndex })));
}
return(Enumerable.Empty <IndexError>());
}
public async Task SaveDefinition_GivenValidYamlAndSearchDoesContainsExistingItemWithNoUpdates_ThenDatasetDefinitionSavedInCosmosAndSearchNotUpdatedAndReturnsOK()
{
//Arrange
string yaml = CreateRawDefinition();
string definitionId = "9183";
byte[] byteArray = Encoding.UTF8.GetBytes(yaml);
MemoryStream stream = new MemoryStream(byteArray);
IHeaderDictionary headerDictionary = new HeaderDictionary();
headerDictionary
.Add("yaml-file", new StringValues(yamlFile));
HttpRequest request = Substitute.For <HttpRequest>();
request
.Headers
.Returns(headerDictionary);
request
.Body
.Returns(stream);
ILogger logger = CreateLogger();
HttpStatusCode statusCode = HttpStatusCode.Created;
IDatasetRepository datasetsRepository = CreateDataSetsRepository();
datasetsRepository
.SaveDefinition(Arg.Any <DatasetDefinition>())
.Returns(statusCode);
DatasetDefinitionIndex existingIndex = new DatasetDefinitionIndex()
{
Description = "14/15 description",
Id = "9183",
LastUpdatedDate = new DateTimeOffset(2018, 6, 19, 14, 10, 2, TimeSpan.Zero),
ModelHash = "DFBD0E1ACD29CEBCF5AD45674688D3780D916294C4DF878074AFD01B67BF129C",
Name = "14/15",
ProviderIdentifier = "None",
};
ISearchRepository <DatasetDefinitionIndex> searchRepository = CreateDatasetDefinitionSearchRepository();
searchRepository
.SearchById(Arg.Is(definitionId))
.Returns(existingIndex);
byte[] excelAsBytes = new byte[100];
IExcelWriter <DatasetDefinition> excelWriter = CreateExcelWriter();
excelWriter
.Write(Arg.Any <DatasetDefinition>())
.Returns(excelAsBytes);
ICloudBlob blob = Substitute.For <ICloudBlob>();
IBlobClient blobClient = CreateBlobClient();
blobClient
.GetBlockBlobReference(Arg.Any <string>())
.Returns(blob);
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService();
definitionChangesDetectionService
.DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Any <DatasetDefinition>())
.Returns(datasetDefinitionChanges);
DefinitionsService service = CreateDefinitionsService(logger, datasetsRepository, searchRepository, excelWriter: excelWriter, blobClient: blobClient, definitionChangesDetectionService: definitionChangesDetectionService);
//Act
IActionResult result = await service.SaveDefinition(request);
//Assert
result
.Should()
.BeOfType <OkResult>();
await searchRepository
.Received(1)
.SearchById(Arg.Is(definitionId));
await searchRepository
.Received(0)
.Index(Arg.Any <IEnumerable <DatasetDefinitionIndex> >());
await datasetsRepository
.Received(1)
.SaveDefinition(Arg.Is <DatasetDefinition>(
i => i.Description == "14/15 description" &&
i.Id == "9183" &&
i.Name == "14/15"
));
logger
.Received(1)
.Information(Arg.Is($"Successfully saved file: {yamlFile} to cosmos db"));
}
public void SearchIndexTest_GivenSearchSchemasAndModels_EnsureFieldsMatch()
{
// Arrange
IList <string> ErrorLog = new List <string>();
DatasetDefinitionIndex datasetDefinitionIndex = new DatasetDefinitionIndex();
ProvidersIndex providersIndex = new ProvidersIndex();
PublishedFundingIndex publishedfundingindex = new PublishedFundingIndex();
PublishedProviderIndex publishedProviderIndex = new PublishedProviderIndex();
SpecificationIndex specificationindex = new SpecificationIndex();
TemplateIndex templateIndex = new TemplateIndex();
IEnumerable <Type> searchIndexTypes = GetTypesWithSearchIndexAttribute();
IEnumerable <string> indexNames = Directory
.GetDirectories(searchIndexDirectoryPath, "*index", SearchOption.TopDirectoryOnly)
.Select(m => new DirectoryInfo(m).Name);
//Act
foreach (string indexName in indexNames)
{
try
{
string jsonFilePath = $@"{searchIndexDirectoryPath}\{indexName}\{indexName}.json";
string jsonText = File.ReadAllText(jsonFilePath, Encoding.UTF8);
SearchIndexSchema searchIndexSchema = JsonConvert.DeserializeObject <SearchIndexSchema>(jsonText);
if (searchIndexSchema?.Name == null)
{
ErrorLog.Add(string.IsNullOrWhiteSpace(jsonText)
? $"{indexName} json is blank"
: $"{indexName} json name is not available");
}
else if (searchIndexSchema.Name != indexName)
{
ErrorLog.Add($"Expected to find index { indexName }, but found { searchIndexSchema.Name }");
}
else
{
Type searchIndexType = searchIndexTypes
.FirstOrDefault(m => m.CustomAttributes
.FirstOrDefault(p => p.NamedArguments
.Any(n => n.TypedValue.Value.ToString() == searchIndexSchema.Name)) != null);
IEnumerable <string> searchIndexProperties = searchIndexType.GetProperties()
.Select(m => m.CustomAttributes
.FirstOrDefault(a => a.AttributeType.Name == "JsonPropertyAttribute")
?.ConstructorArguments[0].Value.ToString().ToLower())
.Where(p => p != null);
IEnumerable <string> searchIndexJsonProperties = searchIndexSchema.Fields
.Select(m => m.Name.ToLower())
.Where(p => p != null);
if (!searchIndexProperties.Any())
{
ErrorLog.Add($"Index {indexName}: The model contains no properties");
}
else if (!searchIndexJsonProperties.Any())
{
ErrorLog.Add($"Index {indexName}: The json contains no properties");
}
else
{
IEnumerable <string> notInJson = searchIndexProperties.Except(searchIndexJsonProperties);
if (notInJson.Any())
{
string properties = string.Join(",", notInJson);
ErrorLog.Add($"Index {indexName}: The model contains the following properties not found in the json schema ({properties})");
}
IEnumerable <string> notInModel = searchIndexJsonProperties.Except(searchIndexProperties);
if (notInModel.Any())
{
string properties = string.Join(",", notInModel);
ErrorLog.Add($"Index {indexName}: The json schema contains the following properties not found in the model ({properties})");
}
}
}
}
catch (Exception e)
{
ErrorLog.Add($"Unexpected error checking '{indexName}': {e.Message}{Environment.NewLine}{ e.StackTrace }");
}
}
//Assert
if (ErrorLog.Any())
{
Assert.Fail(string.Join(Environment.NewLine, ErrorLog));
}
}
public void SearchIndexTest_GivenSearchSchemasAndModels_EnsuresAttributesMatch()
{
//Arrange
IList <string> ErrorLog = new List <string>();
DatasetDefinitionIndex datasetDefinitionIndex = new DatasetDefinitionIndex();
ProvidersIndex providersIndex = new ProvidersIndex();
PublishedFundingIndex publishedfundingindex = new PublishedFundingIndex();
PublishedProviderIndex publishedProviderIndex = new PublishedProviderIndex();
SpecificationIndex specificationindex = new SpecificationIndex();
TemplateIndex templateIndex = new TemplateIndex();
IEnumerable <Type> searchIndexTypes = GetTypesWithSearchIndexAttribute();
IEnumerable <string> indexNames = Directory
.GetDirectories(searchIndexDirectoryPath, "*index", SearchOption.TopDirectoryOnly)
.Select(m => new DirectoryInfo(m).Name);
//Act
foreach (string indexName in indexNames)
{
try
{
string jsonFilePath = $@"{searchIndexDirectoryPath}\{indexName}\{indexName}.json";
string jsonText = File.ReadAllText(jsonFilePath, Encoding.UTF8);
SearchIndexSchema searchIndexSchema = JsonConvert.DeserializeObject <SearchIndexSchema>(jsonText);
if (searchIndexSchema?.Name == null)
{
ErrorLog.Add(string.IsNullOrWhiteSpace(jsonText)
? $"{indexName} json is blank"
: $"{indexName} json name is not available");
}
else if (searchIndexSchema.Name != indexName)
{
ErrorLog.Add($"Expected to find index {indexName}, but found {searchIndexSchema.Name}");
}
else
{
Type searchIndexType = searchIndexTypes.FirstOrDefault(m =>
m.CustomAttributes.FirstOrDefault(p =>
p.NamedArguments.Any(n =>
n.TypedValue.Value.ToString() == searchIndexSchema.Name)) != null);
IEnumerable <PropertyInfo> searchIndexProperties = searchIndexType.GetProperties();
foreach (SearchIndexField searchIndexField in searchIndexSchema.Fields)
{
PropertyInfo matchedProperty = searchIndexProperties.FirstOrDefault(m =>
string.Equals(m.CustomAttributes
.SingleOrDefault(x => x.AttributeType.Name == "JsonPropertyAttribute")
?.ConstructorArguments[0].ToString().Replace("\"", "")
?? m.Name,
searchIndexField.Name,
StringComparison.InvariantCultureIgnoreCase));
if (matchedProperty == null)
{
ErrorLog.Add($"{indexName}: {searchIndexField.Name} did not match any properties");
}
else
{
CheckIndexFieldTypes(matchedProperty, searchIndexField, ErrorLog, indexName);
CheckIndexAttributes(matchedProperty, searchIndexField, ErrorLog, indexName);
}
}
}
}
catch (Exception e)
{
ErrorLog.Add($"Unexpected error checking '{indexName}': {e.Message}{Environment.NewLine}{e.StackTrace}");
}
}
//Assert
if (ErrorLog.Any())
{
Assert.Fail(string.Join(Environment.NewLine, ErrorLog));
}
}