public async Task TabularProcessing_ValidCsv_GeneratesExpectedFileAggregate()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var file = await Session.Get <TabularFile>(FileId);
file.Should().NotBeNull();
file.Should().ShouldBeEquivalentTo(new
{
Id = FileId,
Type = FileType.Tabular,
Bucket = JohnId.ToString(),
BlobId = BlobId,
OwnedBy = JohnId,
CreatedBy = JohnId,
CreatedDateTime = DateTimeOffset.UtcNow,
UpdatedBy = JohnId,
UpdatedDateTime = DateTimeOffset.UtcNow,
ParentId = JohnId,
FileName = blobInfo.FileName,
Length = blobInfo.Length,
Md5 = blobInfo.MD5,
IsDeleted = false,
Status = FileStatus.Processed
}, options => options
.ExcludingMissingMembers()
);
}
public MsWordProcessing(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessFile(JohnId.ToString(), "Developing Standard Approaches for Curating Small Molecule Pharmaceuticals_Jan18_2013.doc", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public async Task ChemicalProcessing_ValidMol_GenerateExpectedSubstanceAggregate()
{
var recordId = Harness.GetProcessedRecords(FileId).First();
var record = await Session.Get <Substance>(recordId);
record.Should().NotBeNull();
record.ShouldBeEquivalentTo(new
{
Id = recordId,
RecordType = RecordType.Structure,
Bucket = JohnId.ToString(),
OwnedBy = JohnId,
CreatedBy = JohnId,
CreatedDateTime = DateTimeOffset.UtcNow,
UpdatedBy = JohnId,
UpdatedDateTime = DateTimeOffset.UtcNow,
ParentId = FileId,
Status = RecordStatus.Processed,
Index = 0,
//Issues = new List<Generic.Domain.ValueObjects.Issue>() { new Generic.Domain.ValueObjects.Issue { Code = "Code", AuxInfo = "AuxInfo", Message = "Message", Severity = Severity.Information, Title = "Title" } }
Issues = new List <Generic.Domain.ValueObjects.Issue>()
{
}
}, options => options
.ExcludingMissingMembers()
);
record.Images.Should().NotBeNullOrEmpty();
record.Images.Should().ContainSingle();
record.Fields.Should().NotBeNullOrEmpty();
record.Fields.Should().HaveCount(3);
record.Properties.Should().NotBeNullOrEmpty();
record.Properties.Should().HaveCount(9);
record.BlobId.Should().NotBeEmpty();
}
public CsvProcessing(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessFile(JohnId.ToString(), "FocusSynthesis_InStock.csv", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public async Task MSWordUpload_ValidDoc_GenerateExpectedFileEntity()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var fileEntityResponse = await JohnApi.GetFileEntityById(FileId);
var fileEntity = JsonConvert.DeserializeObject <JObject>(await fileEntityResponse.Content.ReadAsStringAsync());
fileEntity.Should().NotBeNull();
fileEntity.Should().ContainsJson($@"
{{
'id': '{FileId}',
'blob': {{
'id': '{blobInfo.Id}',
'bucket': '{JohnId}',
'length': {blobInfo.Length},
'md5': '{blobInfo.MD5}'
}},
'subType': '{FileType.Office}',
'ownedBy': '{JohnId}',
'createdBy': '{JohnId}',
'createdDateTime': '{DateTime.UtcNow}',
'updatedBy': '{JohnId}',
'updatedDateTime': '{DateTime.UtcNow}',
'parentId': '{JohnId}',
'name': '{blobInfo.FileName}',
'status': '{FileStatus.Processed}',
'version': *EXIST*
}}" );
fileEntity["images"].Should().NotBeNull();
fileEntity["images"].Should().HaveCount(3);
}
public InvalidMolProcessing(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessRecordsFile(JohnId.ToString(), "ringcount_0.mol", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public ValidCdxProcessing(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessRecordsFile(JohnId.ToString(), "125_11Mos.cdx", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public async Task ReactionProcessing_ValidRxn_GenerateExpectedRactionAggregate()
{
var recordId = Harness.GetProcessedRecords(FileId).First();
var record = await Session.Get <Reaction>(recordId);
record.Should().NotBeNull();
record.Should().BeEquivalentTo(new
{
Id = recordId,
RecordType = RecordType.Reaction,
Bucket = JohnId.ToString(),
OwnedBy = JohnId,
CreatedBy = JohnId,
CreatedDateTime = DateTimeOffset.UtcNow,
UpdatedBy = JohnId,
UpdatedDateTime = DateTimeOffset.UtcNow,
ParentId = FileId,
Status = RecordStatus.Processed,
Index = 0,
Fields = new Field[] {
new Field("Field1", "Value1"),
new Field("Field2", "Value2")
},
Issues = new Issue[] { }
}, options => options
.ExcludingMissingMembers()
);
record.Images.Should().NotBeNullOrEmpty();
record.Images.Should().HaveCount(1);
record.Properties.Should().NotBeNull();
record.Properties.Should().HaveCount(0);
}
public TrainOneModelAndFailDuringTheTraining(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FolderId = TrainModel(JohnId.ToString(), "drugbank_10_records.sdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }, { "case", "train one model and fail during the training" }
}).Result;
}
public PdfProcessing(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessFile(JohnId.ToString(), "Abdelaziz A Full_manuscript.pdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public UploadOneModel(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
var modelBlobId = UploadModel(JohnId.ToString(), "Bernoulli_Naive_Bayes_with_isotonic_class_weights.sav",
new Dictionary <string, object>()
{
{ "parentId", JohnId },
{ "FileType", "MachineLearningModel" },
{ "ModelInfo", new Dictionary <string, object>()
{
{ "Dataset", new Dataset("title", "description") },
{ "Property", new Property("category", "name", "units", "description") },
{ "ModelName", "Some model name" },
{ "Method", "NaiveBayes" },
{ "MethodDisplayName", "Naive Bayes" },
{ "ClassName", "Soluble" },
{ "TestDatasetSize", 0.2 },
{ "KFold", 4 },
{ "Scaler", "scaler" },
{ "Fingerprints", new List <Fingerprint>()
{
new Fingerprint {
Type = "ecfp", Size = 1024, Radius = 3
}
} }
} }
}).Result;
var modelView = Models.Find(new BsonDocument("Blob._id", modelBlobId)).FirstOrDefault() as IDictionary <string, object>;
ModelId = modelView["_id"].As <Guid>();
}
private async Task <Guid> AddFile(bool isPublic, string fileName, Guid parentId)
{
var fileId = ProcessRecordsFile(JohnId.ToString(), fileName, new Dictionary <string, object>()
{
{ "parentId", parentId }
}).Result;
await ChangeShareFile(fileId, isPublic);
var recordsId = await Records.FindAsync(new BsonDocument("FileId", fileId))
.GetAwaiter()
.GetResult()
.ToListAsync();
foreach (var record in recordsId)
{
if (!_testFixture.InternalIds.Contains((Guid)record._id))
{
_testFixture.InternalIds.Add(record._id);
}
}
// InternalIds.AddRange(recordsId.Select(x => (Guid)x._id));
if (!_testFixture.InternalIds.Contains((Guid)fileId))
{
_testFixture.InternalIds.Add(fileId);
}
return(fileId);
}
public InvalidSdfProcessing(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessRecordsFile(JohnId.ToString(), "invalid_sdf_with_20_records_where_first_and_second_are_invalid.sdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public SharingModelTrained(OsdrWebTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FolderId = TrainModel(JohnId.ToString(), "combined lysomotrophic.sdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }, { "case", "valid one model with success optimization" }
}, true).Result;
}
public InvalidCifProcessing(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessRecordsFile(JohnId.ToString(), "1100110_modified.cif", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public SdfProcessingWithOneValidAndOneInvalidRecords(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessRecordsFile(JohnId.ToString(), "test_solubility.sdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public PredictPropertiesValidCase(OsdrTestHarness fixture) : base(fixture)
{
FolderId = PredictProperties(JohnId.ToString(), "combined lysomotrophic.sdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public async Task ReactionProcessing_InvalidRxn_GeneratesExpectedFileAggregate()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var file = await Session.Get <RecordsFile.Domain.RecordsFile>(FileId);
file.Should().NotBeNull();
file.ShouldBeEquivalentTo(new
{
Id = FileId,
Type = FileType.Records,
Bucket = JohnId.ToString(),
BlobId = BlobId,
OwnedBy = JohnId,
CreatedBy = JohnId,
CreatedDateTime = DateTimeOffset.UtcNow,
UpdatedBy = JohnId,
UpdatedDateTime = DateTimeOffset.UtcNow,
ParentId = JohnId,
FileName = blobInfo.FileName,
Length = blobInfo.Length,
Md5 = blobInfo.MD5,
IsDeleted = false,
Status = FileStatus.Failed,
TotalRecords = 0
}, options => options
.ExcludingMissingMembers()
.Using <DateTimeOffset>(ctx => ctx.Subject.Should().BeCloseTo(ctx.Expectation, 5000)).WhenTypeIs <DateTimeOffset>()
);
file.Images.Should().BeEmpty();
}
public async Task CrystalProcessing_ValidCif_GenerateExceptedRecordAggregate()
{
var recordId = Harness.GetProcessedRecords(FileId).First();
var record = await Session.Get <Crystal>(recordId);
record.Should().NotBeNull();
record.Should().BeEquivalentTo(new
{
Id = recordId,
RecordType = RecordType.Crystal,
Bucket = JohnId.ToString(),
OwnedBy = JohnId,
CreatedBy = JohnId,
CreatedDateTime = DateTimeOffset.UtcNow,
UpdatedBy = JohnId,
UpdatedDateTime = DateTimeOffset.UtcNow,
ParentId = FileId,
Status = RecordStatus.Processed,
Index = 0,
Fields = new Field[] {
new Field("Field1", "Value1"),
new Field("Field2", "Value2")
},
Issues = new List <Generic.Domain.ValueObjects.Issue>()
}, options => options
.ExcludingMissingMembers()
);
record.Images.Should().NotBeNullOrEmpty();
record.Images.Should().HaveCount(3);
record.Properties.Should().NotBeNull();
record.Properties.Should().HaveCount(0);
}
public ArchiveGzProcessing(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessFile(JohnId.ToString(), "IMG_0109.gz", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public TrainOneValidModelWithFailedOptimization(OsdrWebTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FolderId = TrainModel(JohnId.ToString(), "combined lysomotrophic.sdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }, { "case", "train model with failed optimization" }
}, true).Result;
}
public async Task Stream_GetStreamUnauthorizedUser_ExpectedOneStream()
{
var response = await UnauthorizedApi.GetStreamFileEntityById(FileId, 0, 1);
var streams = JArray.Parse(await response.Content.ReadAsStringAsync());
streams.Should().NotBeEmpty();
streams.Should().HaveCount(1);
var stream = streams.First();
stream.Should().ContainsJson($@"
{{
'name': 'FileCreated',
'namespace': 'Sds.Osdr.Generic.Domain.Events.Files',
'event': {{
'bucket': '{JohnId.ToString()}',
'blobId': '{BlobId}',
'parentId': '{JohnId}',
'fileName': 'Aspirin.mol',
'fileStatus': 'Loaded',
'fileType': 'Records',
'id': '{FileId}',
'userId': '{JohnId}',
'version': 1
}}
}}" );
}
public ValidSdfProcessing(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FileId = ProcessRecordsFile(JohnId.ToString(), "drugbank_10_records.sdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }
}).Result;
}
public TrainOneValidModelReverseEvents(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FolderId = TrainModel(JohnId.ToString(), "combined lysomotrophic.sdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }, { "case", "valid one model (reverse events order)" }
}).Result;
}
public TrainTwoValidModels(OsdrTestHarness fixture, ITestOutputHelper output) : base(fixture, output)
{
FolderId = TrainModel(JohnId.ToString(), "combined lysomotrophic.sdf", new Dictionary <string, object>()
{
{ "parentId", JohnId }, { "case", "two valid models" }
}).Result;
}
public async Task ImageProcessing_ValidJpg_GeneratesAppropriateModels()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var file = await Session.Get <File>(FileId);
file.Should().NotBeNull();
file.ShouldBeEquivalentTo(new
{
Id = FileId,
Type = FileType.Image,
Bucket = JohnId.ToString(),
BlobId = BlobId,
PdfBucket = file.Bucket,
OwnedBy = JohnId,
CreatedBy = JohnId,
CreatedDateTime = DateTimeOffset.UtcNow,
UpdatedBy = JohnId,
UpdatedDateTime = DateTimeOffset.UtcNow,
ParentId = JohnId,
FileName = blobInfo.FileName,
Length = blobInfo.Length,
Md5 = blobInfo.MD5,
IsDeleted = false,
Status = FileStatus.Processed
}, options => options
.ExcludingMissingMembers()
);
file.Images.Should().NotBeNullOrEmpty();
file.Images.Should().HaveCount(3);
}
public async Task ReactionProcessing_InvalidRnx_GenerateExpectedFileNode()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var fileNodeResponse = await JohnApi.GetNodeById(FileId);
var fileNode = JsonConvert.DeserializeObject <JObject>(await fileNodeResponse.Content.ReadAsStringAsync());
fileNode.Should().ContainsJson($@"
{{
'id': '{FileId}',
'type': 'File',
'subType': 'Records',
'blob': {{
'id': '{blobInfo.Id}',
'bucket': '{JohnId}',
'length': {blobInfo.Length},
'md5': '{blobInfo.MD5}'
}},
'status': '{FileStatus.Failed}',
'ownedBy': '{JohnId}',
'createdBy':'{JohnId}',
'createdDateTime': '{DateTime.UtcNow}',
'updatedBy': '{JohnId}',
'updatedDateTime': '{DateTime.UtcNow}',
'name': '{blobInfo.FileName}',
'parentId': '{JohnId}',
'version': *EXIST*
}}" );
}
protected async Task <Guid> PredictProperties(string bucket, string fileName, IDictionary <string, object> metadata = null)
{
var blobId = await AddBlob(JohnId.ToString(), fileName, metadata);
Guid predictionFolderId = Guid.NewGuid();
Guid correlationId = Guid.NewGuid();
Guid modelBlobId = Guid.NewGuid();
await Harness.Bus.Publish(new CreatePrediction(
id : predictionFolderId,
correlationId : correlationId,
folderId : predictionFolderId,
datasetBlobId : blobId,
datasetBucket : bucket,
modelBlobId : modelBlobId,
modelBucket : JohnId.ToString(),
userId : JohnId
));
if (!Harness.Published.Select <PropertiesPredictionFinished>(m => m.Context.Message.Id == predictionFolderId).Any())
{
throw new TimeoutException();
}
return(predictionFolderId);
}
public async Task ChemicalProcessing_ValidCdx_GenerateExpectedRecordsFileAggregate()
{
var blobInfo = await BlobStorage.GetFileInfo(BlobId, JohnId.ToString());
blobInfo.Should().NotBeNull();
var file = await Session.Get <RecordsFile.Domain.RecordsFile>(FileId);
file.Should().NotBeNull();
file.ShouldBeEquivalentTo(new
{
Id = FileId,
Type = FileType.Records,
Bucket = JohnId.ToString(),
BlobId = BlobId,
OwnedBy = JohnId,
CreatedBy = JohnId,
CreatedDateTime = DateTimeOffset.UtcNow,
UpdatedBy = JohnId,
UpdatedDateTime = DateTimeOffset.UtcNow,
ParentId = JohnId,
FileName = blobInfo.FileName,
Length = blobInfo.Length,
Md5 = blobInfo.MD5,
IsDeleted = false,
Status = FileStatus.Processed,
TotalRecords = 3,
Fields = new List <string>()
}, options => options
.ExcludingMissingMembers()
);
file.Images.Should().NotBeNullOrEmpty();
file.Images.Count.Should().Be(1);
}
public async Task ChemicalProcessing_ValidCdxWithThreeRecords_GenerateExpectedRecordsEntityAndRecordsNode()
{
var records = Harness.GetProcessedRecords(FileId);
records.Should().HaveCount(3);
foreach (var recordId in records)
{
var recordView = Records.Find(new BsonDocument("_id", recordId)).FirstOrDefault() as IDictionary <string, object>;
recordView.Should().NotBeNull();
var recordBlob = recordView["Blob"];
recordBlob.Should().NotBeNull();
recordBlob.Should().BeAssignableTo <IDictionary <string, object> >();
var recordBlobId = (recordBlob as IDictionary <string, object>)["_id"];
recordBlobId.Should().NotBeNull();
recordBlobId.Should().BeOfType <Guid>();
var index = Convert.ToInt32(recordView["Index"]);
index.Should().BeGreaterOrEqualTo(0);
var record = await Session.Get <Substance>((Guid)recordId);
record.Should().NotBeNull();
record.ShouldBeEquivalentTo(new
{
Id = recordId,
RecordType = RecordType.Structure,
Bucket = JohnId.ToString(),
BlobId = recordBlobId,
OwnedBy = JohnId,
CreatedBy = JohnId,
CreatedDateTime = DateTimeOffset.UtcNow,
UpdatedBy = JohnId,
UpdatedDateTime = DateTimeOffset.UtcNow,
ParentId = FileId,
Status = RecordStatus.Processed,
Index = index,
//Issues = new List<Generic.Domain.ValueObjects.Issue>() { new Generic.Domain.ValueObjects.Issue { Code = "Code", AuxInfo = "AuxInfo", Message = "Message", Severity = Generic.Domain.ValueObjects.Severity.Information, Title = "Title" } }
Issues = new List <Generic.Domain.ValueObjects.Issue>()
{
}
}, options => options
.ExcludingMissingMembers()
);
record.Images.Should().NotBeNullOrEmpty();
record.Images.Should().ContainSingle();
record.Fields.Should().BeEmpty();
record.Properties.Should().NotBeNullOrEmpty();
record.Properties.Should().HaveCount(9);
recordView.Should().EntityShouldBeEquivalentTo(record);
var recordNode = Nodes.Find(new BsonDocument("_id", (Guid)recordId)).FirstOrDefault() as IDictionary <string, object>;
recordNode.Should().NotBeNull();
recordNode.Should().NodeShouldBeEquivalentTo(record);
}
}
