public async Task MlProcessing_ModelTraining_GeneratesOneTabular()
{
var tabulars = Harness.GetDependentFiles(FolderId, FileType.Tabular);
tabulars.Should().NotBeNullOrEmpty();
tabulars.Should().HaveCount(2);
foreach (var tabularId in tabulars)
{
var tabularResponse = await JohnApi.GetFileEntityById(tabularId);
var tabularJson = JToken.Parse(await tabularResponse.Content.ReadAsStringAsync());
tabularJson.Should().ContainsJson($@"
{{
'id': '{tabularId}',
'blob': *EXIST*,
'subType': 'Tabular',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': *EXIST*,
'updatedBy': '{JohnId}',
'updatedDateTime': *EXIST*,
'parentId': '{FolderId}',
'name': 'FocusSynthesis_InStock.csv',
'status': 'Processed',
'version': 7
}}");
}
await Task.CompletedTask;
}
public async Task MlPrediction_SingleCsvFileProcessed()
{
var tabulars = await Fixture.GetDependentFiles(FolderId, FileType.Tabular);
tabulars.Should().NotBeNullOrEmpty();
tabulars.Should().HaveCount(1);
foreach (var tabularId in tabulars)
{
var tabularResponse = await JohnApi.GetFileEntityById(tabularId);
var tabularJson = JToken.Parse(await tabularResponse.Content.ReadAsStringAsync());
tabularJson.Should().ContainsJson($@"
{{
'id': '{tabularId}',
'blob': *EXIST*,
'subType': 'Tabular',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': *EXIST*,
'updatedBy': '{JohnId}',
'updatedDateTime': *EXIST*,
'parentId': '{FolderId}',
'name': 'PropertiesPrediction.csv',
'status': 'Processed',
'version': 7
}}");
}
await Task.CompletedTask;
}
public async Task MlProcessing_ModelTraining_WaitsWhileAllAssociatedGenericFilesProcessed()
{
var models = Harness.GetDependentFilesExcept(FolderId, FileType.Image, FileType.Tabular, FileType.Pdf);
models.Should().HaveCount(1);
foreach (var modelId in models)
{
var modelGenericFiles = Harness.GetDependentFiles(modelId, FileType.Image, FileType.Tabular, FileType.Pdf);
modelGenericFiles.Should().HaveCount(5);
modelGenericFiles.ToList().ForEach(async fileId =>
{
var file = await Session.Get <File>(modelId);
file.Should().NotBeNull();
file.Status.Should().Be(FileStatus.Processed);
});
}
var reportFiles = Harness.GetDependentFiles(FolderId, FileType.Image, FileType.Tabular, FileType.Pdf);
reportFiles.Should().HaveCount(3);
reportFiles.ToList().ForEach(async id =>
{
var fileResponse = await JohnApi.GetFileEntityById(id);
fileResponse.EnsureSuccessStatusCode();
var jsonFile = JToken.Parse(await fileResponse.Content.ReadAsStringAsync());
jsonFile["status"].Should().BeEquivalentTo("Processed");
});
await Task.CompletedTask;
}
public async Task SpectraProcessing_InvalidJdx_GenerateExpectedFileEntity()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var fileEntityResponse = await JohnApi.GetFileEntityById(FileId);
var fileEntity = JsonConvert.DeserializeObject <JObject>(await fileEntityResponse.Content.ReadAsStringAsync());
fileEntity.Should().NotBeNull();
fileEntity.Should().ContainsJson($@"
{{
'id': '{FileId}',
'blob': {{
'id': '{blobInfo.Id}',
'bucket': '{JohnId}',
'length': {blobInfo.Length},
'md5': '{blobInfo.MD5}'
}},
'subType': '{FileType.Records}',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': '{DateTime.UtcNow}',
'updatedBy': '{JohnId}',
'updatedDateTime': '{DateTime.UtcNow}',
'parentId': '{JohnId}',
'name': '{blobInfo.FileName}',
'status': '{FileStatus.Failed}',
'version': *EXIST*
}}" );
}
public async Task ChemicalProcessing_ValidSdf_GenerateExpectedFileEntity()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var fileEntityResponse = await JohnApi.GetFileEntityById(FileId);
var fileEntity = JsonConvert.DeserializeObject <JObject>(await fileEntityResponse.Content.ReadAsStringAsync());
fileEntity.Should().NotBeNull();
fileEntity.Should().ContainsJson($@"
{{
'id': '{FileId}',
'blob': {{
'id': '{blobInfo.Id}',
'bucket': '{JohnId}',
'length': {blobInfo.Length},
'md5': '{blobInfo.MD5}'
}},
'subType': '{FileType.Records}',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': *EXIST*,
'updatedBy': '{JohnId}',
'updatedDateTime': *EXIST*,
'parentId': '{JohnId}',
'name': '{blobInfo.FileName}',
'status': '{FileStatus.Processed}',
'version': 8,
'totalRecords': 2,
'properties': {{
'fields': [
'StdInChI',
'StdInChIKey',
'SMILES'
],
'chemicalProperties': [
'MOST_ABUNDANT_MASS',
'MONOISOTOPIC_MASS',
'MOLECULAR_WEIGHT',
'MOLECULAR_FORMULA',
'SMILES',
'NonStdInChI',
'InChIKey',
'InChI',
'NonStdInChIKey'
]
}}
}}" );
fileEntity["images"].Should().NotBeNull();
fileEntity["images"].Should().HaveCount(1);
}
public async Task BlobImageSharing_WithUnauthorizeUser_ReturnsExpectedNotFound()
{
var fileResponse = await JohnApi.GetFileEntityById(FileId);
var file = JObject.Parse(await fileResponse.Content.ReadAsStringAsync());
var imageId = file["images"].First()["id"].ToObject <Guid>();
var response = await UnauthorizedApi.GetImagesFileEntityById(FileId, imageId);
response.IsSuccessStatusCode.Should().Be(false);
// response.StatusCode.ShouldBeEquivalentTo(HttpStatusCode.NotFound);
// response.ReasonPhrase.ShouldAllBeEquivalentTo("Not Found");
}
public async Task MlProcessing_ModelTraining_GeneratesPdfReport()
{
var pdfFiles = Harness.GetDependentFiles(FolderId, FileType.Pdf);
pdfFiles.Should().NotBeNullOrEmpty();
pdfFiles.Should().HaveCount(1);
foreach (var pdfId in pdfFiles)
{
var pdfResponse = await JohnApi.GetFileEntityById(pdfId);
var pdfJson = JToken.Parse(await pdfResponse.Content.ReadAsStringAsync());
pdfJson.Should().ContainsJson($@"
{{
'id': '{pdfId}',
'blob': *EXIST*,
'subType': 'Pdf',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': *EXIST*,
'updatedBy': '{JohnId}',
'updatedDateTime': *EXIST*,
'parentId': '{FolderId}',
'name': 'ML_report.pdf',
'status': 'Processed',
'version': 7
}}");
pdfJson["images"].Should().HaveCount(3);
var nodePdfResponse = await JohnApi.GetNodeById(pdfId);
var nodeJson = JToken.Parse(await nodePdfResponse.Content.ReadAsStringAsync());
nodeJson.Should().ContainsJson($@"
{{
'id': '{pdfId}',
'blob': *EXIST*,
'subType': 'Pdf',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': *EXIST*,
'updatedBy': '{JohnId}',
'updatedDateTime': *EXIST*,
'parentId': '{FolderId}',
'name': 'ML_report.pdf',
'status': 'Processed',
'version': 7
}}");
}
await Task.CompletedTask;
}
public async Task FileSharing_WithUnauthorizeUser_ReturnsExpectedImage()
{
var fileResponse = await JohnApi.GetFileEntityById(FileId);
var file = JObject.Parse(await fileResponse.Content.ReadAsStringAsync());
var imageId = file["images"].First()["id"].ToObject <Guid>();
var blobResponse = await UnauthorizedApi.GetImagesFileEntityById(FileId, imageId);
blobResponse.EnsureSuccessStatusCode();
blobResponse.StatusCode.ShouldBeEquivalentTo(HttpStatusCode.OK);
blobResponse.Content.Headers.ContentType.MediaType.ShouldBeEquivalentTo("application/octet-stream");
blobResponse.Content.Headers.ContentLength.Should().BeGreaterThan(10000);
blobResponse.Content.Headers.ContentDisposition.FileName.ShouldBeEquivalentTo("Aspirin.mol.svg");
}
public async Task FileSharing_WithAuthorizeUser_ReturnsExpectedImage()
{
var fileResponse = await JohnApi.GetFileEntityById(FileId);
var file = JObject.Parse(await fileResponse.Content.ReadAsStringAsync());
var imageId = file["images"].First()["id"].ToObject <Guid>();
var blobResponse = await JohnApi.GetImagesFileEntityById(FileId, imageId);
blobResponse.EnsureSuccessStatusCode();
blobResponse.StatusCode.Should().Be(HttpStatusCode.OK);
blobResponse.Content.Headers.ContentType.MediaType.Should().Be("application/octet-stream");
blobResponse.Content.Headers.ContentLength.Should().Be(175430);
blobResponse.Content.Headers.ContentDisposition.FileName.Should().Be("Chemical-diagram.png");
}
public async Task ChemicalProcessing_ValidMol_GenerateExpectedFileEntity()
{
var startTime = DateTime.UtcNow;
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var fileEntityResponse = await JohnApi.GetFileEntityById(FileId);
var fileEntity = JsonConvert.DeserializeObject <JObject>(await fileEntityResponse.Content.ReadAsStringAsync());
fileEntity.Should().NotBeNull();
fileEntity.Should().ContainsJson($@"
{{
'id': '{FileId}',
'blob': {{
'id': '{blobInfo.Id}',
'bucket': '{JohnId}',
'length': {blobInfo.Length},
'md5': '{blobInfo.MD5}'
}},
'subType': '{FileType.Records}',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': '{startTime}',
'updatedBy': '{JohnId}',
'updatedDateTime': '{startTime}',
'parentId': '{JohnId}',
'name': '{blobInfo.FileName}',
'status': '{FileStatus.Processed}',
'version': *EXIST*,
'totalRecords': 1,
'properties': {{
'fields': [
'StdInChI',
'StdInChIKey',
'SMILES',
'AuxInfo',
'Formula',
'Mw',
'CSID'
],
'chemicalProperties': *EXIST*
}}
}}" );
}
public async Task Stream_GetStream_ExpectedFullStream()
{
var fileResponse = await JohnApi.GetFileEntityById(FileId);
var file = JToken.Parse(await fileResponse.Content.ReadAsStringAsync());
var version = file["version"].ToObject <int>();
var events = new List <string>()
{
"PermissionsChanged",
"StatusChanged",
"AggregatedPropertiesAdded",
"ImageAdded",
"FieldsAdded",
"TotalRecordsUpdated",
"StatusChanged",
"RecordsFileCreated",
"FileCreated"
};
var response = await JohnApi.GetStreamFileEntityById(FileId, -1, -1);
var streams = JArray.Parse(await response.Content.ReadAsStringAsync());
streams.Should().NotBeEmpty();
streams.Should().HaveCount(9);
for (var i = 0; i < version; i++)
{
var @event = streams[i];
events.Should().Contain(new[] { @event["name"].ToObject <string>() });
var eventVersion = version - i;
@event.Should().ContainsJson($@"
{{
'namespace': *EXIST*,
'event': {{
'id': '{FileId}',
'userId': '{JohnId}',
'version': {eventVersion}
}}
}}" );
}
}
public async Task MlProcessing_TrainOneModelAndFailDuringTheReportGeneration_WaitsWhileAllAssociatedGenericFilesProcessed()
{
var models = Harness.GetDependentFilesExcept(FolderId, FileType.Image, FileType.Tabular, FileType.Pdf);
models.Should().HaveCount(1);
foreach (var modelId in models)
{
var model = await Session.Get <Model>(modelId);
model.Should().NotBeNull();
model.Status.Should().Be(ModelStatus.Processed);
model.Images.Should().HaveCount(3);
}
var files = (Harness.GetDependentFiles(FolderId, FileType.Image, FileType.Tabular, FileType.Pdf)).ToList();
files.Should().HaveCount(2);
files.ToList().ForEach(async fileId =>
{
var fileResponse = await JohnApi.GetFileEntityById(fileId);
fileResponse.EnsureSuccessStatusCode();
var jsonFile = JToken.Parse(await fileResponse.Content.ReadAsStringAsync());
jsonFile["status"].Should().BeEquivalentTo("Processed");
});
await Task.CompletedTask;
//var files = Fixture.GetDependentFiles(FolderId).ToList();
//files.Should().HaveCount(5);
//files.ForEach(async id =>
//{
// var fileResponse = await Api.GetFileEntityById(id);
// fileResponse.EnsureSuccessStatusCode();
// var jsonFile = JToken.Parse(await fileResponse.Content.ReadAsStringAsync());
// jsonFile["status"].Should().BeEquivalentTo("Processed");
//});
//await Task.CompletedTask;
}
public async Task CrystalProcessing_ValidMol_GenerateExpectedFileEntity()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var fileEntityResponse = await JohnApi.GetFileEntityById(FileId);
var fileEntity = JsonConvert.DeserializeObject <JObject>(await fileEntityResponse.Content.ReadAsStringAsync());
fileEntity.Should().NotBeNull();
fileEntity.Should().ContainsJson($@"
{{
'id': '{FileId}',
'blob': {{
'id': '{blobInfo.Id}',
'bucket': '{JohnId}',
'length': {blobInfo.Length},
'md5': '{blobInfo.MD5}'
}},
'subType': '{FileType.Records}',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': '{DateTime.UtcNow}',
'updatedBy': '{JohnId}',
'updatedDateTime': '{DateTime.UtcNow}',
'parentId': '{JohnId}',
'name': '{blobInfo.FileName}',
'status': '{FileStatus.Processed}',
'version': *EXIST*,
'totalRecords': 1,
'properties': {{
'fields': [
'Field1',
'Field2'
]
}}
}}" );
fileEntity["images"].Should().NotBeNull();
fileEntity["images"].Should().HaveCount(3);
}
public async Task MlProcessing_ModelTraining_AllGenericFilesPersisted()
{
var files = Harness.GetDependentFiles(FolderId, FileType.Image, FileType.Tabular, FileType.Pdf);
foreach (var id in files)
{
var fileResponse = JohnApi.GetFileEntityById(id).Result;
var fileJson = JToken.Parse(fileResponse.Content.ReadAsStringAsync().Result);
var fileNodeResponse = JohnApi.GetNodeById(id).Result;
fileNodeResponse.EnsureSuccessStatusCode();
fileJson["status"].ToObject <string>().Should().BeEquivalentTo("Processed");
var jsonNode = JToken.Parse(fileNodeResponse.Content.ReadAsStringAsync().Result);
jsonNode.Should().NotBeEmpty();
jsonNode.Should().ContainsJson($@"
{{
'id': '{fileJson["id"].ToObject<Guid>()}',
'type': 'File',
'subType': *EXIST*,
'blob': *EXIST*,
'status': '{fileJson["status"].ToObject<string>()}',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': *EXIST*,
'updatedBy': '{JohnId}',
'updatedDateTime': *EXIST*,
'name': '{fileJson["name"].ToObject<string>()}',
'parentId': '{fileJson["parentId"].ToObject<Guid>()}',
'version': {fileJson["version"].ToObject<int>()}
}}");
if (fileJson.Contains("images") && jsonNode.Contains("images"))
{
jsonNode["images"].Should().BeEquivalentTo(fileJson["images"]);
}
}
;
await Task.CompletedTask;
}
public async Task MlProcessing_InvalidModelTraining_WaitsWhileAllAssociatedGenericFilesProcessed()
{
var modelId = await Fixture.GetDependentFilesExcept(FolderId, FileType.Image, FileType.Pdf, FileType.Tabular);
modelId.Should().HaveCount(1);
modelId.SingleOrDefault().Should().NotBeEmpty();
var dependentFiles = Fixture.GetDependentFiles(modelId.Single()).ToList();
dependentFiles.Should().HaveCount(2);
dependentFiles.ToList().ForEach(async id =>
{
var fileResponse = await JohnApi.GetFileEntityById(id);
fileResponse.EnsureSuccessStatusCode();
var jsonFile = JToken.Parse(await fileResponse.Content.ReadAsStringAsync());
jsonFile["status"].Should().BeEquivalentTo("Processed");
});
await Task.CompletedTask;
}
public async Task UpdateMetadata_UpdateGenericMetadata_ExpectedRenamedFolder()
{
var processedFile = await Harness.Session.Get <File>(FileId);
var url = $"/api/entities/files/{FileId}?version={processedFile.Version}";
var data = $"[{{'op':'replace','path':'Metadata','value':[{{'name':'test1', 'value': 'value1'}}]}}]";
JohnApi.PatchData(url, data).Wait();
Harness.WaitMetadataUpdated(FileId);
var response = await JohnApi.GetFileEntityById(FileId);
response.EnsureSuccessStatusCode();
var json = JToken.Parse(await response.Content.ReadAsStringAsync());
json.Should().ContainsJson($@"
{{
'id': '{FileId}',
'createdBy': '{JohnId}',
'createdDateTime': *EXIST*,
'updatedBy': '{JohnId}',
'updatedDateTime': *EXIST*,
'ownedBy': '{JohnId}',
'version': 8,
'properties': {{
'metadata':
[
{{
'name': 'test1',
'value': 'value1'
}}
]
}}
}}");
}
public async Task ChemicalProcessing_ValidSdf_GenerateExpectedFileEntity()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var fileEntityResponse = await JohnApi.GetFileEntityById(FileId);
var fileEntity = JsonConvert.DeserializeObject <JObject>(await fileEntityResponse.Content.ReadAsStringAsync());
fileEntity.Should().NotBeNull();
fileEntity.Should().ContainsJson($@"
{{
'id': '{FileId}',
'blob': {{
'id': '{blobInfo.Id}',
'bucket': '{JohnId}',
'length': {blobInfo.Length},
'md5': '{blobInfo.MD5}'
}},
'subType': '{FileType.Records}',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': *EXIST*,
'updatedBy': '{JohnId}',
'updatedDateTime': *EXIST*,
'parentId': '{JohnId}',
'name': '{blobInfo.FileName}',
'status': '{FileStatus.Processed}',
'version': 8,
'totalRecords': 10,
'properties': {{
'fields': [
'DATABASE_ID',
'DATABASE_NAME',
'SMILES',
'INCHI_IDENTIFIER',
'INCHI_KEY',
'FORMULA',
'MOLECULAR_WEIGHT',
'EXACT_MASS',
'JCHEM_ACCEPTOR_COUNT',
'JCHEM_AVERAGE_POLARIZABILITY',
'JCHEM_BIOAVAILABILITY',
'JCHEM_DONOR_COUNT',
'JCHEM_FORMAL_CHARGE',
'JCHEM_GHOSE_FILTER',
'JCHEM_IUPAC',
'ALOGPS_LOGP',
'JCHEM_LOGP',
'ALOGPS_LOGS',
'JCHEM_MDDR_LIKE_RULE',
'JCHEM_NUMBER_OF_RINGS',
'JCHEM_PHYSIOLOGICAL_CHARGE',
'JCHEM_PKA',
'JCHEM_PKA_STRONGEST_ACIDIC',
'JCHEM_PKA_STRONGEST_BASIC',
'JCHEM_POLAR_SURFACE_AREA',
'JCHEM_REFRACTIVITY',
'JCHEM_ROTATABLE_BOND_COUNT',
'JCHEM_RULE_OF_FIVE',
'ALOGPS_SOLUBILITY',
'JCHEM_TRADITIONAL_IUPAC',
'JCHEM_VEBER_RULE',
'DRUGBANK_ID',
'SECONDARY_ACCESSION_NUMBERS',
'DRUG_GROUPS',
'GENERIC_NAME',
'PRODUCTS',
'SALTS',
'SYNONYMS',
'INTERNATIONAL_BRANDS',
'JCHEM_ATOM_COUNT'
],
'chemicalProperties': [
'MOST_ABUNDANT_MASS',
'MONOISOTOPIC_MASS',
'MOLECULAR_WEIGHT',
'MOLECULAR_FORMULA',
'SMILES',
'InChIKey',
'InChI'
]
}}
}}" );
fileEntity["images"].Should().NotBeNull();
fileEntity["images"].Should().HaveCount(1);
}
