public async Task <IActionResult> ExportAsExcel()
{
var entries = await _dispatcher.DispatchAsync(new GetEntititesQuery <ConfigurationEntry>());
using var stream = new MemoryStream();
_configurationEntriesExcelWriter.Write(entries, stream);
return(File(stream.ToArray(), MediaTypeNames.Application.Octet, "ConfigurationEntries.xlsx"));
}
public void ConvertMarkdownToExcel(
[Option("i", "input markdown file path.")] string markdownFilePath,
[Option("o", "output Excel file path.")] string excelFilePath
)
{
var markdown = _markdownReader.ReadToEnd(markdownFilePath);
var usdm = _markdownParser.Parse(markdown);
var book = _excelDecoder.Decode(usdm);
_excelWriter.Write(book, excelFilePath);
}
public async Task SaveDefinition_GivenValidYamlAndDoesContainsExistingItemWithNoModelUpdates_ThenDoesNotAddMessageToTopicAndReturnsOK()
{
//Arrange
string yaml = CreateRawDefinition();
string definitionId = "9183";
byte[] byteArray = Encoding.UTF8.GetBytes(yaml);
MemoryStream stream = new MemoryStream(byteArray);
IHeaderDictionary headerDictionary = new HeaderDictionary();
headerDictionary
.Add("yaml-file", new StringValues(yamlFile));
HttpRequest request = Substitute.For <HttpRequest>();
request
.Headers
.Returns(headerDictionary);
request
.Body
.Returns(stream);
ILogger logger = CreateLogger();
HttpStatusCode statusCode = HttpStatusCode.Created;
DatasetDefinition existingDatasetDefinition = new DatasetDefinition
{
Id = definitionId
};
IDatasetRepository datasetsRepository = CreateDataSetsRepository();
datasetsRepository
.SaveDefinition(Arg.Any <DatasetDefinition>())
.Returns(statusCode);
datasetsRepository
.GetDatasetDefinition(Arg.Is(definitionId))
.Returns(existingDatasetDefinition);
byte[] excelAsBytes = new byte[100];
IExcelWriter <DatasetDefinition> excelWriter = CreateExcelWriter();
excelWriter
.Write(Arg.Any <DatasetDefinition>())
.Returns(excelAsBytes);
ICloudBlob blob = Substitute.For <ICloudBlob>();
IBlobClient blobClient = CreateBlobClient();
blobClient
.GetBlockBlobReference(Arg.Any <string>())
.Returns(blob);
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService();
definitionChangesDetectionService
.DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Is(existingDatasetDefinition))
.Returns(datasetDefinitionChanges);
IMessengerService messengerService = CreateMessengerService();
DefinitionsService service = CreateDefinitionsService(
logger,
datasetsRepository,
excelWriter: excelWriter,
blobClient: blobClient,
definitionChangesDetectionService: definitionChangesDetectionService,
messengerService: messengerService);
//Act
IActionResult result = await service.SaveDefinition(request);
//Assert
result
.Should()
.BeOfType <OkResult>();
await
messengerService
.DidNotReceive()
.SendToTopic(
Arg.Is(ServiceBusConstants.TopicNames.DataDefinitionChanges),
Arg.Any <DatasetDefinitionChanges>(),
Arg.Any <IDictionary <string, string> >());
}
public async Task SaveDefinition_GivenValidYamlAndSearchDoesContainsExistingItemWithNoUpdates_ThenDatasetDefinitionSavedInCosmosAndSearchNotUpdatedAndReturnsOK()
{
//Arrange
string yaml = CreateRawDefinition();
string definitionId = "9183";
byte[] byteArray = Encoding.UTF8.GetBytes(yaml);
MemoryStream stream = new MemoryStream(byteArray);
IHeaderDictionary headerDictionary = new HeaderDictionary();
headerDictionary
.Add("yaml-file", new StringValues(yamlFile));
HttpRequest request = Substitute.For <HttpRequest>();
request
.Headers
.Returns(headerDictionary);
request
.Body
.Returns(stream);
ILogger logger = CreateLogger();
HttpStatusCode statusCode = HttpStatusCode.Created;
IDatasetRepository datasetsRepository = CreateDataSetsRepository();
datasetsRepository
.SaveDefinition(Arg.Any <DatasetDefinition>())
.Returns(statusCode);
DatasetDefinitionIndex existingIndex = new DatasetDefinitionIndex()
{
Description = "14/15 description",
Id = "9183",
LastUpdatedDate = new DateTimeOffset(2018, 6, 19, 14, 10, 2, TimeSpan.Zero),
ModelHash = "DFBD0E1ACD29CEBCF5AD45674688D3780D916294C4DF878074AFD01B67BF129C",
Name = "14/15",
ProviderIdentifier = "None",
};
ISearchRepository <DatasetDefinitionIndex> searchRepository = CreateDatasetDefinitionSearchRepository();
searchRepository
.SearchById(Arg.Is(definitionId))
.Returns(existingIndex);
byte[] excelAsBytes = new byte[100];
IExcelWriter <DatasetDefinition> excelWriter = CreateExcelWriter();
excelWriter
.Write(Arg.Any <DatasetDefinition>())
.Returns(excelAsBytes);
ICloudBlob blob = Substitute.For <ICloudBlob>();
IBlobClient blobClient = CreateBlobClient();
blobClient
.GetBlockBlobReference(Arg.Any <string>())
.Returns(blob);
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService();
definitionChangesDetectionService
.DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Any <DatasetDefinition>())
.Returns(datasetDefinitionChanges);
DefinitionsService service = CreateDefinitionsService(logger, datasetsRepository, searchRepository, excelWriter: excelWriter, blobClient: blobClient, definitionChangesDetectionService: definitionChangesDetectionService);
//Act
IActionResult result = await service.SaveDefinition(request);
//Assert
result
.Should()
.BeOfType <OkResult>();
await searchRepository
.Received(1)
.SearchById(Arg.Is(definitionId));
await searchRepository
.Received(0)
.Index(Arg.Any <IEnumerable <DatasetDefinitionIndex> >());
await datasetsRepository
.Received(1)
.SaveDefinition(Arg.Is <DatasetDefinition>(
i => i.Description == "14/15 description" &&
i.Id == "9183" &&
i.Name == "14/15"
));
logger
.Received(1)
.Information(Arg.Is($"Successfully saved file: {yamlFile} to cosmos db"));
}
async public Task SaveDefinition_GivenValidYamlAndSearchDoesNotContainExistingItem_ThenSaveWasSuccesfulAndReturnsOK()
{
//Arrange
string yaml = CreateRawDefinition();
string definitionId = "9183";
byte[] byteArray = Encoding.UTF8.GetBytes(yaml);
MemoryStream stream = new MemoryStream(byteArray);
IHeaderDictionary headerDictionary = new HeaderDictionary();
headerDictionary
.Add("yaml-file", new StringValues(yamlFile));
HttpRequest request = Substitute.For <HttpRequest>();
request
.Headers
.Returns(headerDictionary);
request
.Body
.Returns(stream);
ILogger logger = CreateLogger();
HttpStatusCode statusCode = HttpStatusCode.Created;
IDatasetRepository datasetsRepository = CreateDataSetsRepository();
datasetsRepository
.SaveDefinition(Arg.Any <DatasetDefinition>())
.Returns(statusCode);
ISearchRepository <DatasetDefinitionIndex> searchRepository = CreateDatasetDefinitionSearchRepository();
searchRepository
.SearchById(Arg.Is(definitionId))
.Returns((DatasetDefinitionIndex)null);
byte[] excelAsBytes = new byte[100];
IExcelWriter <DatasetDefinition> excelWriter = CreateExcelWriter();
excelWriter
.Write(Arg.Any <DatasetDefinition>())
.Returns(excelAsBytes);
ICloudBlob blob = Substitute.For <ICloudBlob>();
IBlobClient blobClient = CreateBlobClient();
blobClient
.GetBlockBlobReference(Arg.Any <string>())
.Returns(blob);
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService();
definitionChangesDetectionService
.DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Any <DatasetDefinition>())
.Returns(datasetDefinitionChanges);
DefinitionsService service = CreateDefinitionsService(logger, datasetsRepository, searchRepository,
excelWriter: excelWriter, blobClient: blobClient, definitionChangesDetectionService: definitionChangesDetectionService);
//Act
IActionResult result = await service.SaveDefinition(request);
//Assert
result
.Should()
.BeOfType <OkResult>();
await searchRepository
.Received(1)
.SearchById(Arg.Is(definitionId));
await searchRepository
.Received(1)
.Index(Arg.Is <IEnumerable <DatasetDefinitionIndex> >(
i => i.First().Description == "14/15 description" &&
i.First().Id == "9183" &&
!string.IsNullOrWhiteSpace(i.First().ModelHash) &&
i.First().Name == "14/15" &&
i.First().ProviderIdentifier == "None"
));
await datasetsRepository
.Received(1)
.SaveDefinition(Arg.Is <DatasetDefinition>(
i => i.Description == "14/15 description" &&
i.Id == "9183" &&
i.Name == "14/15"
));
logger
.Received(1)
.Information(Arg.Is($"Successfully saved file: {yamlFile} to cosmos db"));
}
async public Task SaveDefinition_GivenValidYamlButFailsToUploadToBlobStorage_ReturnsInvalidServerError()
{
//Arrange
string yaml = CreateRawDefinition();
string definitionId = "9183";
byte[] byteArray = Encoding.UTF8.GetBytes(yaml);
MemoryStream stream = new MemoryStream(byteArray);
IHeaderDictionary headerDictionary = new HeaderDictionary();
headerDictionary
.Add("yaml-file", new StringValues(yamlFile));
HttpRequest request = Substitute.For <HttpRequest>();
request
.Headers
.Returns(headerDictionary);
request
.Body
.Returns(stream);
ILogger logger = CreateLogger();
HttpStatusCode statusCode = HttpStatusCode.Created;
IDatasetRepository datasetsRepository = CreateDataSetsRepository();
datasetsRepository
.SaveDefinition(Arg.Any <DatasetDefinition>())
.Returns(statusCode);
ISearchRepository <DatasetDefinitionIndex> searchRepository = CreateDatasetDefinitionSearchRepository();
searchRepository
.SearchById(Arg.Is(definitionId))
.Returns((DatasetDefinitionIndex)null);
byte[] excelAsBytes = new byte[100];
IExcelWriter <DatasetDefinition> excelWriter = CreateExcelWriter();
excelWriter
.Write(Arg.Any <DatasetDefinition>())
.Returns(excelAsBytes);
ICloudBlob blob = Substitute.For <ICloudBlob>();
blob
.When(x => x.UploadFromStreamAsync(Arg.Any <Stream>()))
.Do(x => { throw new Exception($"Failed to upload 14/15 blob storage"); });
IBlobClient blobClient = CreateBlobClient();
blobClient
.GetBlockBlobReference(Arg.Is("schemas/14_15.xlsx"))
.Returns(blob);
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService();
definitionChangesDetectionService
.DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Any <DatasetDefinition>())
.Returns(datasetDefinitionChanges);
DefinitionsService service = CreateDefinitionsService(logger, datasetsRepository, searchRepository,
excelWriter: excelWriter, blobClient: blobClient, definitionChangesDetectionService: definitionChangesDetectionService);
//Act
IActionResult result = await service.SaveDefinition(request);
//Assert
result
.Should()
.BeOfType <InternalServerErrorResult>()
.Which
.Value
.Should()
.Be($"Failed to upload 14/15 blob storage");
}
async public Task SaveDefinition_GivenValidYamlButFailsToGenerateExcelFile_ReturnsInvalidServerError()
{
//Arrange
string yaml = CreateRawDefinition();
string definitionId = "9183";
byte[] byteArray = Encoding.UTF8.GetBytes(yaml);
MemoryStream stream = new MemoryStream(byteArray);
IHeaderDictionary headerDictionary = new HeaderDictionary();
headerDictionary
.Add("yaml-file", new StringValues(yamlFile));
HttpRequest request = Substitute.For <HttpRequest>();
request
.Headers
.Returns(headerDictionary);
request
.Body
.Returns(stream);
ILogger logger = CreateLogger();
HttpStatusCode statusCode = HttpStatusCode.Created;
IDatasetRepository datasetsRepository = CreateDataSetsRepository();
datasetsRepository
.SaveDefinition(Arg.Any <DatasetDefinition>())
.Returns(statusCode);
ISearchRepository <DatasetDefinitionIndex> searchRepository = CreateDatasetDefinitionSearchRepository();
searchRepository
.SearchById(Arg.Is(definitionId))
.Returns((DatasetDefinitionIndex)null);
byte[] excelAsBytes = new byte[0];
IExcelWriter <DatasetDefinition> excelWriter = CreateExcelWriter();
excelWriter
.Write(Arg.Any <DatasetDefinition>())
.Returns(excelAsBytes);
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService();
definitionChangesDetectionService
.DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Any <DatasetDefinition>())
.Returns(datasetDefinitionChanges);
DefinitionsService service = CreateDefinitionsService(logger, datasetsRepository, searchRepository, excelWriter: excelWriter, definitionChangesDetectionService: definitionChangesDetectionService);
//Act
IActionResult result = await service.SaveDefinition(request);
//Assert
result
.Should()
.BeOfType <InternalServerErrorResult>()
.Which
.Value
.Should()
.Be($"Failed to generate excel file for 14/15");
logger
.Received(1)
.Error(Arg.Is($"Failed to generate excel file for 14/15"));
}
async public Task <IActionResult> SaveDefinition(HttpRequest request)
{
string yaml = await request.GetRawBodyStringAsync();
string yamlFilename = request.GetYamlFileNameFromRequest();
if (string.IsNullOrEmpty(yaml))
{
_logger.Error($"Null or empty yaml provided for file: {yamlFilename}");
return(new BadRequestObjectResult($"Invalid yaml was provided for file: {yamlFilename}"));
}
var deserializer = new DeserializerBuilder()
.WithNamingConvention(new CamelCaseNamingConvention())
.Build();
DatasetDefinition definition = null;
try
{
definition = deserializer.Deserialize <DatasetDefinition>(yaml);
}
catch (Exception exception)
{
_logger.Error(exception, $"Invalid yaml was provided for file: {yamlFilename}");
return(new BadRequestObjectResult($"Invalid yaml was provided for file: {yamlFilename}"));
}
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
DatasetDefinition existingDefinition = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(definition.Id));
if (existingDefinition != null)
{
datasetDefinitionChanges = _definitionChangesDetectionService.DetectChanges(definition, existingDefinition);
IEnumerable <string> relationships = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDistinctRelationshipSpecificationIdsForDatasetDefinitionId(datasetDefinitionChanges.Id));
IEnumerable <FieldDefinitionChanges> fieldDefinitionChanges = datasetDefinitionChanges.TableDefinitionChanges.SelectMany(m => m.FieldChanges);
if (!relationships.IsNullOrEmpty() && !fieldDefinitionChanges.IsNullOrEmpty())
{
if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.RemovedField)))
{
return(new BadRequestObjectResult("Unable to remove a field as there are currently relationships setup against this schema"));
}
if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.IdentifierType)))
{
return(new BadRequestObjectResult("Unable to change provider identifier as there are currently relationships setup against this schema"));
}
}
}
try
{
HttpStatusCode result = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.SaveDefinition(definition));
if (!result.IsSuccess())
{
int statusCode = (int)result;
_logger.Error($"Failed to save yaml file: {yamlFilename} to cosmos db with status {statusCode}");
return(new StatusCodeResult(statusCode));
}
await IndexDatasetDefinition(definition);
}
catch (Exception exception)
{
_logger.Error(exception, $"Exception occurred writing to yaml file: {yamlFilename} to cosmos db");
return(new InternalServerErrorResult($"Exception occurred writing to yaml file: {yamlFilename} to cosmos db"));
}
byte[] excelAsBytes = _excelWriter.Write(definition);
if (excelAsBytes == null || excelAsBytes.Length == 0)
{
_logger.Error($"Failed to generate excel file for {definition.Name}");
return(new InternalServerErrorResult($"Failed to generate excel file for {definition.Name}"));
}
try
{
await SaveToBlobStorage(excelAsBytes, definition.Name);
}
catch (Exception ex)
{
return(new InternalServerErrorResult(ex.Message));
}
_logger.Information($"Successfully saved file: {yamlFilename} to cosmos db");
if (existingDefinition != null && datasetDefinitionChanges.HasChanges)
{
IDictionary <string, string> properties = request.BuildMessageProperties();
await _messengerService.SendToTopic(ServiceBusConstants.TopicNames.DataDefinitionChanges, datasetDefinitionChanges, properties);
}
return(new OkResult());
}
