public int Run(IRDMPPlatformRepositoryServiceLocator repositoryLocator, IDataLoadEventListener listener, ICheckNotifier checkNotifier, GracefulCancellationToken token)
{
// if we have no listener use a throw immediately one (generate exceptions if it went badly)
if (listener == null)
{
listener = new ThrowImmediatelyDataLoadEventListener();
}
// whatever happens we want a listener to record the worst result for the return code (even if theres ignore all errors listeners being used)
var toMemory = new ToMemoryDataLoadEventListener(false);
// User might have some additional listeners registered
listener = new ForkDataLoadEventListener(AdditionalListeners.Union(new [] { toMemory, listener }).ToArray());
// build the engine and run it
var engine = UseCase.GetEngine(Pipeline, listener);
engine.ExecutePipeline(token ?? new GracefulCancellationToken());
// return code of -1 if it went badly otherwise 0
var exitCode = toMemory.GetWorst() >= ProgressEventType.Error ? -1:0;
if (exitCode == 0)
{
PipelineExecutionFinishedsuccessfully?.Invoke(this, new PipelineEngineEventArgs(engine));
}
return(exitCode);
}
public void TestSerialPipelineExecution()
{
// set SetUp two engines, one with a locked cache progress/load schedule
// run the serial execution and ensure that only one engine had its 'ExecutePipeline' method called
var engine1 = new Mock <IDataFlowPipelineEngine>();
var engine2 = new Mock <IDataFlowPipelineEngine>();
var tokenSource = new GracefulCancellationTokenSource();
var listener = new ThrowImmediatelyDataLoadEventListener();
// set SetUp the engine map
var loadProgress1 = Mock.Of <ILoadProgress>();
var loadProgress2 = Mock.Of <ILoadProgress>();
// set SetUp the lock provider
var engineMap = new Dictionary <IDataFlowPipelineEngine, ILoadProgress>
{
{ engine1.Object, loadProgress1 },
{ engine2.Object, loadProgress2 }
};
// create the execution object
var pipelineExecutor = new SerialPipelineExecution();
// Act
pipelineExecutor.Execute(new [] { engine1.Object, engine2.Object }, tokenSource.Token, listener);
// engine1 should have been executed once
engine1.Verify(e => e.ExecutePipeline(It.IsAny <GracefulCancellationToken>()), Times.Once);
// engine2 should also have been run (locking isn't a thing anymore)
engine2.Verify(e => e.ExecutePipeline(It.IsAny <GracefulCancellationToken>()), Times.Once);
}
public void CommittingNewCohortFile_CallPipeline()
{
var listener = new ThrowImmediatelyDataLoadEventListener();
var proj = new Project(DataExportRepository, projName);
proj.ProjectNumber = 999;
proj.SaveToDatabase();
CohortCreationRequest request = new CohortCreationRequest(proj, new CohortDefinition(null, "CommittingNewCohorts", 1, 999, _externalCohortTable), (DataExportRepository)DataExportRepository, "fish");
request.Check(new ThrowImmediatelyCheckNotifier());
DelimitedFlatFileDataFlowSource source = new DelimitedFlatFileDataFlowSource();
BasicCohortDestination destination = new BasicCohortDestination();
source.Separator = ",";
source.StronglyTypeInput = true;
DataFlowPipelineEngine <DataTable> pipeline = new DataFlowPipelineEngine <DataTable>((DataFlowPipelineContext <DataTable>)request.GetContext(), source, destination, listener);
pipeline.Initialize(new FlatFileToLoad(new FileInfo(filename)), request);
pipeline.ExecutePipeline(new GracefulCancellationToken());
//there should be a new ExtractableCohort now
Assert.NotNull(request.NewCohortDefinition.ID);
var ec = DataExportRepository.GetAllObjects <ExtractableCohort>().Single(c => c.OriginID == request.NewCohortDefinition.ID);
//with the data in it from the test file
Assert.AreEqual(ec.Count, 3);
}
public void CreateAndUpdateZip()
{
var _dir = TestContext.CurrentContext.WorkDirectory;
var _zt = new ZipTestLayout(new DirectoryInfo(_dir), "yyyy-MM-dd", CacheArchiveType.Zip, CacheFileGranularity.Hour, new NoSubdirectoriesCachePathResolver());
var _listener = new ThrowImmediatelyDataLoadEventListener();
var when = DateTime.Now;
var targetzip = _zt.GetArchiveFileInfoForDate(when, _listener);
List <FileInfo> files = new List <FileInfo>();
// First create a zip file with one item in
File.Delete(targetzip.FullName);
files.Add(new FileInfo(Path.Combine(_dir, Path.GetRandomFileName())));
using (var sw = new StreamWriter(files[0].FullName))
sw.WriteLine("Example data file");
_zt.ArchiveFiles(files.ToArray(), when, _listener);
using (var zip = ZipFile.Open(targetzip.FullName, ZipArchiveMode.Read))
Assert.True(zip.Entries.Count == 1);
// Create a second file and add that to the zip too
files.Add(new FileInfo(Path.Combine(_dir, Path.GetRandomFileName())));
using (var sw = new StreamWriter(files[1].FullName))
sw.WriteLine("Another example data file");
_zt.ArchiveFiles(files.ToArray(), when, _listener);
using (var zip = ZipFile.Open(targetzip.FullName, ZipArchiveMode.Read))
Assert.True(zip.Entries.Count == 2);
// Re-add just the first file: resulting zip should still contain both files
_zt.ArchiveFiles(files.GetRange(0, 1).ToArray(), when, _listener);
using (var zip = ZipFile.Open(targetzip.FullName, ZipArchiveMode.Read))
Assert.True(zip.Entries.Count == 2);
files.ForEach(s => File.Delete(s.FullName));
File.Delete(targetzip.FullName);
}
private DiscoveredTable UploadTestDataAsTableToServer(DatabaseType type, out Catalogue catalogue, out ExtractionInformation[] extractionInformations, out TableInfo tableinfo)
{
var listener = new ThrowImmediatelyDataLoadEventListener();
var db = GetCleanedServer(type);
var data = GetTestDataTable();
var uploader = new DataTableUploadDestination();
uploader.PreInitialize(db, listener);
uploader.ProcessPipelineData(data, listener, new GracefulCancellationToken());
uploader.Dispose(listener, null);
var tbl = db.ExpectTable(uploader.TargetTableName);
Assert.IsTrue(tbl.Exists());
ColumnInfo[] cis;
new TableInfoImporter(CatalogueRepository, tbl).DoImport(out tableinfo, out cis);
CatalogueItem[] cataitems;
new ForwardEngineerCatalogue(tableinfo, cis, true).ExecuteForwardEngineering(out catalogue, out cataitems, out extractionInformations);
return(tbl);
}
public void TestRoundRobinPipelineExecution()
{
// set SetUp two engines, one with a locked cache progress/load schedule
// run the serial execution and ensure that only one engine had its 'ExecutePipeline' method called
var engine1 = new Mock <IDataFlowPipelineEngine>();
var engine2 = new Mock <IDataFlowPipelineEngine>();
var tokenSource = new GracefulCancellationTokenSource();
var listener = new ThrowImmediatelyDataLoadEventListener();
// first time both engines return that they have more data, second time they are both complete
engine1.SetupSequence(engine => engine.ExecuteSinglePass(It.IsAny <GracefulCancellationToken>()))
.Returns(true)
.Returns(false)
.Throws <InvalidOperationException>();
engine2.SetupSequence(engine => engine.ExecuteSinglePass(It.IsAny <GracefulCancellationToken>()))
.Returns(true)
.Returns(false)
.Throws <InvalidOperationException>();
// set SetUp the engine map
var loadProgress1 = Mock.Of <ILoadProgress>();
var loadProgress2 = Mock.Of <ILoadProgress>();
// set SetUp the lock provider
var engineMap = new Dictionary <IDataFlowPipelineEngine, ILoadProgress>
{
{ engine1.Object, loadProgress1 },
{ engine2.Object, loadProgress2 }
};
// create the execution object
var pipelineExecutor = new RoundRobinPipelineExecution();
// Act
pipelineExecutor.Execute(new[] { engine1.Object, engine2.Object }, tokenSource.Token, listener);
// Assert
// engine1 should have been executed once
engine1.Verify();
// engine2 should not have been executed as it is locked
engine1.Verify();
}
private DiscoveredTable UploadTestDataAsTableToServer(DatabaseType type, out ICatalogue catalogue, out ExtractionInformation[] extractionInformations, out ITableInfo tableinfo)
{
var listener = new ThrowImmediatelyDataLoadEventListener();
var db = GetCleanedServer(type);
var data = GetTestDataTable();
var uploader = new DataTableUploadDestination();
uploader.PreInitialize(db, listener);
uploader.ProcessPipelineData(data, listener, new GracefulCancellationToken());
uploader.Dispose(listener, null);
var tbl = db.ExpectTable(uploader.TargetTableName);
Assert.IsTrue(tbl.Exists());
catalogue = Import(tbl, out tableinfo, out _, out _, out extractionInformations);
return(tbl);
}
protected void Execute(out ExtractionPipelineUseCase pipelineUseCase, out IExecuteDatasetExtractionDestination results, IDataLoadEventListener listener = null)
{
if (listener == null)
{
listener = new ThrowImmediatelyDataLoadEventListener();
}
DataLoadInfo d = new DataLoadInfo("Internal", _testDatabaseName, "IgnoreMe", "", true, new DiscoveredServer(UnitTestLoggingConnectionString));
Pipeline pipeline = null;
//because extractable columns is likely to include chi column, it will be removed from the collection (for a substitution identifier)
var before = _extractableColumns.ToArray();
try
{
pipeline = SetupPipeline();
pipelineUseCase = new ExtractionPipelineUseCase(_request.Configuration.Project, _request, pipeline, d);
pipelineUseCase.Execute(listener);
Assert.IsNotEmpty(pipelineUseCase.Source.Request.QueryBuilder.SQL);
Assert.IsTrue(pipelineUseCase.ExtractCommand.State == ExtractCommandState.Completed);
}
finally
{
if (pipeline != null)
{
pipeline.DeleteInDatabase();
}
}
results = pipelineUseCase.Destination;
_extractableColumns = new List <IColumn>(before);
}
private DataTable GetDataTable(IDataLoadEventListener listener)
{
if (listener == null)
{
listener = new ThrowImmediatelyDataLoadEventListener();
}
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to lookup which server to interrogate for CohortIdentificationConfiguration " + _cohortIdentificationConfiguration));
if (_cohortIdentificationConfiguration.RootCohortAggregateContainer_ID == null)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "CohortIdentificationConfiguration '" + _cohortIdentificationConfiguration + "' has no RootCohortAggregateContainer_ID, is it empty?"));
}
var cohortCompiler = new CohortCompiler(_cohortIdentificationConfiguration);
ICompileable rootContainerTask;
//no caching set up so no point in running CohortCompilerRunner
if (_cohortIdentificationConfiguration.QueryCachingServer_ID == null)
{
rootContainerTask = RunRootContainerOnlyNoCaching(cohortCompiler);
}
else
{
rootContainerTask = RunAllTasksWithRunner(cohortCompiler, listener);
}
if (rootContainerTask.State == CompilationState.Executing)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Root container task was unexpectedly still executing... let's give it a little longer to run"));
int countdown = Math.Max(5000, Timeout * 1000);
while (rootContainerTask.State == CompilationState.Executing && countdown > 0)
{
Task.Delay(100).Wait();
countdown -= 100;
}
}
if (rootContainerTask.State != CompilationState.Finished)
{
throw new Exception("CohortIdentificationCriteria execution resulted in state '" + rootContainerTask.State + "'", rootContainerTask.CrashMessage);
}
if (rootContainerTask == null)
{
throw new Exception("Root container task was null, was the execution cancelled? / crashed");
}
var execution = cohortCompiler.Tasks[rootContainerTask];
if (execution.Identifiers == null || execution.Identifiers.Rows.Count == 0)
{
throw new Exception("CohortIdentificationCriteria execution resulted in an empty dataset (there were no cohorts matched by the query?)");
}
var dt = execution.Identifiers;
foreach (DataColumn column in dt.Columns)
{
column.ReadOnly = false;
}
return(dt);
}
public void TestBasicDataTableAnonymiser5(LoggerTestCase testCase)
{
//Create a names table that will go into the database
var dt = new DataTable();
dt.Columns.Add("Name");
dt.Rows.Add(new[] { "Thomas" });
dt.Rows.Add(new[] { "Wallace" });
dt.Rows.Add(new[] { "Frank" });
//upload the DataTable from memory into the database
var discoveredTable = GetCleanedServer(DatabaseType.MicrosoftSQLServer).CreateTable("ForbiddenNames", dt);
try
{
TableInfo tableInfo;
//import the persistent TableInfo reference
var importer = Import(discoveredTable, out tableInfo, out _);
//Create the test dataset chunks that will be anonymised
var dtStories1 = new DataTable();
dtStories1.Columns.Add("Story");
dtStories1.Rows.Add(new[] { "Thomas went to school regularly" }); //1st redact
dtStories1.Rows.Add(new[] { "It seems like Wallace went less regularly" }); //2nd redact
dtStories1.Rows.Add(new[] { "Mr Smitty was the teacher" });
var dtStories2 = new DataTable();
dtStories2.Columns.Add("Story");
dtStories2.Rows.Add(new[] { "Things were going so well" });
dtStories2.Rows.Add(new[] { "And then it all turned bad for Wallace" }); //3rd redact
var dtStories3 = new DataTable();
dtStories3.Columns.Add("Story");
dtStories3.Rows.Add(new[] { "There were things creeping in the dark" });
dtStories3.Rows.Add(new[] { "Surely Frank would know what to do. Frank was a genius" }); //4th redact
dtStories3.Rows.Add(new[] { "Mr Smitty was the teacher" });
//Create the anonymiser
var a = new BasicDataTableAnonymiser5();
//Tell it about the database table
a.NamesTable = tableInfo;
//Create a listener according to the test case
IDataLoadEventListener listener = null;
switch (testCase)
{
case LoggerTestCase.ToConsole:
listener = new ThrowImmediatelyDataLoadEventListener();
break;
case LoggerTestCase.ToMemory:
listener = new ToMemoryDataLoadEventListener(true);
break;
case LoggerTestCase.ToDatabase:
//get the default logging server
var logManager = CatalogueRepository.GetDefaultLogManager();
//create a new super task Anonymising Data Tables
logManager.CreateNewLoggingTaskIfNotExists("Anonymising Data Tables");
//setup a listener that goes to this logging database
listener = new ToLoggingDatabaseDataLoadEventListener(this, logManager, "Anonymising Data Tables", "Run on " + DateTime.Now);
break;
default:
throw new ArgumentOutOfRangeException("testCase");
}
//run the anonymisation
//process all 3 batches
a.ProcessPipelineData(dtStories1, listener, new GracefulCancellationToken());
a.ProcessPipelineData(dtStories2, listener, new GracefulCancellationToken());
a.ProcessPipelineData(dtStories3, listener, new GracefulCancellationToken());
//check the results
switch (testCase)
{
case LoggerTestCase.ToMemory:
Assert.AreEqual(4, ((ToMemoryDataLoadEventListener)listener).LastProgressRecieivedByTaskName["REDACTING Names"].Progress.Value);
break;
case LoggerTestCase.ToDatabase:
((ToLoggingDatabaseDataLoadEventListener)listener).FinalizeTableLoadInfos();
break;
}
}
finally
{
//finally drop the database table
discoveredTable.Drop();
}
}
