public static async Task <Guid> PredictProperties(this OsdrTestHarness harness, string bucket, string fileName, IDictionary <string, object> metadata = null)
{
var blobId = await harness.AddBlob(harness.JohnId.ToString(), fileName, metadata);
Guid predictionFolderId = Guid.NewGuid();
Guid correlationId = Guid.NewGuid();
Guid modelBlobId = Guid.NewGuid();
await harness.BusControl.Publish(new CreatePrediction(
id : predictionFolderId,
correlationId : correlationId,
folderId : predictionFolderId,
datasetBlobId : blobId,
datasetBucket : bucket,
modelBlobId : modelBlobId,
modelBucket : harness.JohnId.ToString(),
userId : harness.JohnId
));
if (!harness.Received.Select <MachineLearning.Sagas.Events.PropertiesPredictionFinished>(m => m.Context.Message.Id == predictionFolderId).Any())
{
throw new TimeoutException();
}
return(predictionFolderId);
}
public static async Task <Guid> UploadModel(this OsdrTestHarness harness, string bucket, string fileName, IDictionary <string, object> metadata = null)
{
var blobId = await harness.AddBlob(bucket, fileName, metadata);
var blobInfo = await harness.BlobStorage.GetFileInfo(blobId, bucket);
await harness.BusControl.Publish <BlobLoaded>(new
{
BlobInfo = new LoadedBlobInfo(blobId, fileName, blobInfo.Length, harness.JohnId, blobInfo.UploadDateTime, blobInfo.MD5, bucket, blobInfo.Metadata),
TimeStamp = DateTimeOffset.UtcNow
});
if (!harness.Received.Select <MachineLearning.Domain.Events.ModelPersisted>(e => e.Context.Message.BlobId == blobId).Any())
{
throw new TimeoutException();
}
var modelProcessed = harness.Received.Select <MachineLearning.Domain.Events.ModelPersisted>(m => m.Context.Message.BlobId == blobId).First();
return(modelProcessed.Context.Message.Id);
}
public static async Task <Guid> ProcessFile(this OsdrTestHarness harness, string bucket, string fileName, IDictionary <string, object> metadata = null)
{
Log.Information($"Processing started [{fileName}]");
var blobId = await harness.AddBlob(bucket, fileName, metadata);
var blobInfo = await harness.BlobStorage.GetFileInfo(blobId, bucket);
await harness.BusControl.Publish <BlobLoaded>(new
{
BlobInfo = new LoadedBlobInfo(blobId, fileName, blobInfo.Length, harness.JohnId, blobInfo.UploadDateTime, blobInfo.MD5, bucket, blobInfo.Metadata),
TimeStamp = DateTimeOffset.UtcNow
});
if (!harness.Received.Select <Generic.Sagas.Events.FileProcessed>(m => m.Context.Message.BlobId == blobId).Any())
{
throw new TimeoutException();
}
var fileProcessed = harness.Received.Select <Generic.Sagas.Events.FileProcessed>(m => m.Context.Message.BlobId == blobId).First();
var fileId = fileProcessed.Context.Message.Id;
if (!harness.Received.Select <Generic.Domain.Events.Files.NodeStatusPersisted>(m => (m.Context.Message.Status == Generic.Domain.FileStatus.Processed || m.Context.Message.Status == Generic.Domain.FileStatus.Failed) && m.Context.Message.Id == fileId).Any())
{
throw new TimeoutException();
}
if (!harness.Received.Select <Generic.Domain.Events.Files.StatusPersisted>(m => (m.Context.Message.Status == Generic.Domain.FileStatus.Processed || m.Context.Message.Status == Generic.Domain.FileStatus.Failed) && m.Context.Message.Id == fileId).Any())
{
throw new TimeoutException();
}
var events = await harness.CqrsEventStore.Get(fileId, 0);
Log.Information($"Aggregate-{fileId}: {string.Join(", ", events.Select(e => $"{e.GetType().Name} ({e.Version})"))}");
Log.Information($"Processing finished [{fileName}]");
return(fileId);
}
public static async Task <Guid> TrainModel(this OsdrTestHarness harness, string bucket, string fileName, IDictionary <string, object> metadata = null)
{
var blobId = await harness.AddBlob(harness.JohnId.ToString(), fileName, metadata);
Guid correlationId = NewId.NextGuid();
var modelFolderId = NewId.NextGuid();
if (metadata["case"].Equals("valid one model with success optimization") ||
metadata["case"].Equals("train model with failed optimization"))
{
await harness.BusControl.Publish <StartTraining>(new
{
Id = modelFolderId,
ParentId = modelFolderId,
SourceBlobId = blobId,
SourceBucket = bucket,
CorrelationId = correlationId,
UserId = harness.JohnId,
SourceFileName = fileName,
Methods = new List <string>(new[]
{
"NaiveBayes"
}),
ClassName = "ClassName",
Optimize = true
});
}
else if (!metadata["case"].Equals("two valid models"))
{
await harness.BusControl.Publish <StartTraining>(new
{
Id = modelFolderId,
ParentId = modelFolderId,
SourceBlobId = blobId,
SourceBucket = bucket,
CorrelationId = correlationId,
UserId = harness.JohnId,
SourceFileName = fileName,
Scaler = "Somebody knows what is Scaler???",
Methods = new List <string>(new[]
{
"NaiveBayes"
}),
ClassName = "ClassName",
SubSampleSize = (decimal)0.2,
TestDataSize = (decimal)0.2,
KFold = 4,
Fingerprints = new List <IDictionary <string, object> >()
{
new Dictionary <string, object>()
{
{ "radius", 3 },
{ "size", 1024 },
{ "type", FingerprintType.ecfp }
}
},
Optimize = false
});
}
if (metadata["case"].Equals("two valid models"))
{
await harness.BusControl.Publish <StartTraining>(new
{
Id = modelFolderId,
ParentId = modelFolderId,
SourceBlobId = blobId,
SourceBucket = bucket,
CorrelationId = correlationId,
UserId = harness.JohnId,
SourceFileName = fileName,
Scaler = "Somebody knows what is Scaler???",
Methods = new List <string>(new[]
{
"NaiveBayes",
"LogisticRegression"
}),
ClassName = "ClassName",
SubSampleSize = (decimal)0.2,
TestDataSize = (decimal)0.2,
KFold = 4,
Fingerprints = new List <IDictionary <string, object> >()
{
new Dictionary <string, object>()
{
{ "radius", 3 },
{ "size", 1024 },
{ "type", FingerprintType.ecfp }
}
},
Optimize = false
});
}
if (!harness.Received.Select <MachineLearning.Domain.Events.TrainingFinished>(m => m.Context.Message.Id == modelFolderId).Any())
{
throw new TimeoutException();
}
return(modelFolderId);
}
public static async Task <Guid> ProcessRecordsFile(this OsdrTestHarness harness, string bucket, string fileName, IDictionary <string, object> metadata = null)
{
Log.Information($"Processing started [{fileName}]");
var blobId = await harness.AddBlob(bucket, fileName, metadata.ToDictionary(c => c.Key, c => c.Value, StringComparer.OrdinalIgnoreCase));
var blobInfo = await harness.BlobStorage.GetFileInfo(blobId, bucket);
await harness.BusControl.Publish <BlobLoaded>(new
{
BlobInfo = new LoadedBlobInfo(blobId, fileName, blobInfo.Length, (Guid)metadata["parentId"], blobInfo.UploadDateTime, blobInfo.MD5, bucket, blobInfo.Metadata),
TimeStamp = DateTimeOffset.UtcNow
});
if (!harness.Received.Select <RecordsFile.Sagas.Events.RecordsFileProcessed>(m => m.Context.Message.BlobId == blobId).Any())
{
throw new TimeoutException();
}
var fileProcessed = harness.Received.Select <RecordsFile.Sagas.Events.RecordsFileProcessed>(m => m.Context.Message.BlobId == blobId).First();
var fileId = fileProcessed.Context.Message.Id;
if (fileProcessed.Context.Message.ProcessedRecords != harness.GetProcessedRecords(fileId).Count())
{
throw new Exception($"fileProcessed.Context.Message.ProcessedRecords ({fileProcessed.Context.Message.ProcessedRecords}) != Fixture.GetProcessedRecords(fileId).Count() ({harness.GetProcessedRecords(fileId).Count()})");
}
foreach (var recordId in harness.GetProcessedRecords(fileId))
{
if (!harness.Received.Select <RecordsFile.Domain.Events.Records.NodeStatusPersisted>(m => m.Context.Message.Status == RecordsFile.Domain.RecordStatus.Processed && m.Context.Message.Id == recordId).Any())
{
throw new TimeoutException();
}
if (!harness.Received.Select <RecordsFile.Domain.Events.Records.StatusPersisted>(m => m.Context.Message.Status == RecordsFile.Domain.RecordStatus.Processed && m.Context.Message.Id == recordId).Any())
{
throw new TimeoutException();
}
}
if (fileProcessed.Context.Message.FailedRecords != harness.GetInvalidRecords(fileId).Count())
{
throw new Exception("fileProcessed.Context.Message.FailedRecords != Fixture.GetInvalidRecords(fileId).Count()");
}
foreach (var recordId in harness.GetInvalidRecords(fileId))
{
if (!harness.Received.Select <RecordsFile.Domain.Events.Records.RecordPersisted>(m => m.Context.Message.Id == recordId).Any())
{
throw new TimeoutException();
}
if (!harness.Received.Select <RecordsFile.Domain.Events.Records.NodeRecordPersisted>(m => m.Context.Message.Id == recordId).Any())
{
throw new TimeoutException();
}
}
if (!harness.Received.Select <Generic.Domain.Events.Files.NodeStatusPersisted>(m => (m.Context.Message.Status == Generic.Domain.FileStatus.Processed || m.Context.Message.Status == Generic.Domain.FileStatus.Failed) && m.Context.Message.Id == fileId).Any())
{
throw new TimeoutException();
}
if (!harness.Received.Select <Generic.Domain.Events.Files.StatusPersisted>(m => (m.Context.Message.Status == Generic.Domain.FileStatus.Processed || m.Context.Message.Status == Generic.Domain.FileStatus.Failed) && m.Context.Message.Id == fileId).Any())
{
throw new TimeoutException();
}
var events = await harness.CqrsEventStore.Get(fileId, 0);
Log.Information($"Aggregate-{fileId}: {string.Join(", ", events.Select(e => $"{e.GetType().Name} ({e.Version})"))}");
Log.Information($"Processing finished [{fileName}]");
return(fileId);
}
