protected ConfigureAndExecutePipelineUI GetConfigureAndExecuteControl(CohortCreationRequest request, string description)
{
var catalogueRepository = Activator.RepositoryLocator.CatalogueRepository;
ConfigureAndExecutePipelineUI configureAndExecuteDialog = new ConfigureAndExecutePipelineUI(request, Activator);
configureAndExecuteDialog.Dock = DockStyle.Fill;
configureAndExecuteDialog.PipelineExecutionFinishedsuccessfully += (o, args) => OnCohortCreatedSuccessfully(configureAndExecuteDialog, request);
//add in the logging server
var loggingServer = catalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
if (loggingServer != null)
{
var logManager = new LogManager(loggingServer);
logManager.CreateNewLoggingTaskIfNotExists(ExtractableCohort.CohortLoggingTask);
//create a db listener
var toDbListener = new ToLoggingDatabaseDataLoadEventListener(this, logManager, ExtractableCohort.CohortLoggingTask, description);
//make all messages go to both the db and the UI
configureAndExecuteDialog.SetAdditionalProgressListener(toDbListener);
//after executing the pipeline finalise the db listener table info records
configureAndExecuteDialog.PipelineExecutionFinishedsuccessfully += (s, e) => toDbListener.FinalizeTableLoadInfos();
}
return(configureAndExecuteDialog);
}
protected override void ExecuteRun(object runnable, OverrideSenderIDataLoadEventListener listener)
{
var dataLoadInfo = StartAudit();
var globalCommand = runnable as ExtractGlobalsCommand;
var datasetCommand = runnable as ExtractDatasetCommand;
var logging = new ToLoggingDatabaseDataLoadEventListener(_logManager, dataLoadInfo);
var fork =
datasetCommand != null ?
new ForkDataLoadEventListener(logging, listener, new ElevateStateListener(datasetCommand)):
new ForkDataLoadEventListener(logging, listener);
if (globalCommand != null)
{
var useCase = new ExtractionPipelineUseCase(_activator, _project, _globalsCommand, _pipeline, dataLoadInfo)
{
Token = Token
};
useCase.Execute(fork);
}
if (datasetCommand != null)
{
var executeUseCase = new ExtractionPipelineUseCase(_activator, _project, datasetCommand, _pipeline, dataLoadInfo)
{
Token = Token
};
executeUseCase.Execute(fork);
}
logging.FinalizeTableLoadInfos();
dataLoadInfo.CloseAndMarkComplete();
}
public int Run(IRDMPPlatformRepositoryServiceLocator repositoryLocator, IDataLoadEventListener listener, ICheckNotifier checkNotifier, GracefulCancellationToken token)
{
const string dataLoadTask = "caching";
CacheProgress cp = repositoryLocator.CatalogueRepository.GetObjectByID <CacheProgress>(_options.CacheProgress);
var defaults = repositoryLocator.CatalogueRepository.GetServerDefaults();
var loggingServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
if (loggingServer == null)
{
throw new NotSupportedException("No default logging server specified, you must specify one in ");
}
var logManager = new LogManager(loggingServer);
logManager.CreateNewLoggingTaskIfNotExists(dataLoadTask);
switch (_options.Command)
{
case CommandLineActivity.run:
//Setup dual listeners for the Cache process, one ticks the lifeline one very message and one logs to the logging db
var toLog = new ToLoggingDatabaseDataLoadEventListener(this, logManager, dataLoadTask, "Caching " + cp);
var forkListener = new ForkDataLoadEventListener(toLog, listener);
try
{
var cachingHost = new CachingHost(repositoryLocator.CatalogueRepository);
cachingHost.RetryMode = _options.RetryMode;
cachingHost.CacheProgressList = new ICacheProgress[] { cp }.ToList(); //run the cp
//By default caching host will block
cachingHost.TerminateIfOutsidePermissionWindow = true;
cachingHost.Start(forkListener, token);
}
finally
{
//finish everything
toLog.FinalizeTableLoadInfos();
}
break;
case CommandLineActivity.check:
var checkable = new CachingPreExecutionChecker(cp);
checkable.Check(checkNotifier);
break;
default:
throw new ArgumentOutOfRangeException();
}
return(0);
}
private void StartAuditIfExists(string tableName)
{
if (LoggingServer != null)
{
_loggingDatabaseSettings = DataAccessPortal.GetInstance().ExpectServer(LoggingServer, DataAccessContext.Logging);
var logManager = new LogManager(_loggingDatabaseSettings);
logManager.CreateNewLoggingTaskIfNotExists("Internal");
_dataLoadInfo = (DataLoadInfo)logManager.CreateDataLoadInfo("Internal", GetType().Name, "Loading table " + tableName, "", false);
_loggingDatabaseListener = new ToLoggingDatabaseDataLoadEventListener(logManager, _dataLoadInfo);
}
}
public override void GenerateReport(ICatalogue c, IDataLoadEventListener listener, CancellationToken cancellationToken)
{
SetupLogging(c.CatalogueRepository);
var toDatabaseLogger = new ToLoggingDatabaseDataLoadEventListener(this, _logManager, _loggingTask, "DQE evaluation of " + c);
var forker = new ForkDataLoadEventListener(listener, toDatabaseLogger);
try
{
_catalogue = c;
var dqeRepository = new DQERepository(c.CatalogueRepository);
byPivotCategoryCubesOverTime.Add("ALL", new PeriodicityCubesOverTime("ALL"));
byPivotRowStatesOverDataLoadRunId.Add("ALL", new DQEStateOverDataLoadRunId("ALL"));
Check(new FromDataLoadEventListenerToCheckNotifier(forker));
var sw = Stopwatch.StartNew();
using (var con = _server.GetConnection())
{
con.Open();
var cmd = _server.GetCommand(_queryBuilder.SQL, con);
cmd.CommandTimeout = 500000;
var t = cmd.ExecuteReaderAsync(cancellationToken);
t.Wait(cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException("User cancelled DQE while fetching data");
}
var r = t.Result;
int progress = 0;
while (r.Read())
{
cancellationToken.ThrowIfCancellationRequested();
progress++;
int dataLoadRunIDOfCurrentRecord = 0;
//to start with assume we will pass the results for the 'unknown batch' (where data load run ID is null or not available)
//if the DataReader is likely to have a data load run ID column
if (_containsDataLoadID)
{
//get data load run id
int?runID = dqeRepository.ObjectToNullableInt(r[_dataLoadRunFieldName]);
//if it has a value use it (otherwise it is null so use 0 - ugh I know, it's a primary key constraint issue)
if (runID != null)
{
dataLoadRunIDOfCurrentRecord = (int)runID;
}
}
string pivotValue = null;
//if the user has a pivot category configured
if (_pivotCategory != null)
{
pivotValue = GetStringValueForPivotField(r[_pivotCategory], forker);
if (!haveComplainedAboutNullCategories && string.IsNullOrWhiteSpace(pivotValue))
{
forker.OnNotify(this,
new NotifyEventArgs(ProgressEventType.Warning,
"Found a null/empty value for pivot category '" + _pivotCategory +
"', this record will ONLY be recorded under ALL and not it's specific category, you will not be warned of further nulls because there are likely to be many if there are any"));
haveComplainedAboutNullCategories = true;
pivotValue = null;
}
}
//always increase the "ALL" category
ProcessRecord(dqeRepository, dataLoadRunIDOfCurrentRecord, r,
byPivotCategoryCubesOverTime["ALL"], byPivotRowStatesOverDataLoadRunId["ALL"]);
//if there is a value in the current record for the pivot column
if (pivotValue != null)
{
//if it is a novel
if (!byPivotCategoryCubesOverTime.ContainsKey(pivotValue))
{
//we will need to expand the dictionaries
if (byPivotCategoryCubesOverTime.Keys.Count > MaximumPivotValues)
{
throw new OverflowException(
"Encountered more than " + MaximumPivotValues + " values for the pivot column " + _pivotCategory +
" this will result in crazy space usage since it is a multiplicative scale of DQE tesseracts");
}
//expand both the time periodicity and the state results
byPivotRowStatesOverDataLoadRunId.Add(pivotValue,
new DQEStateOverDataLoadRunId(pivotValue));
byPivotCategoryCubesOverTime.Add(pivotValue, new PeriodicityCubesOverTime(pivotValue));
}
//now we are sure that the dictionaries have the category field we can increment it
ProcessRecord(dqeRepository, dataLoadRunIDOfCurrentRecord, r,
byPivotCategoryCubesOverTime[pivotValue], byPivotRowStatesOverDataLoadRunId[pivotValue]);
}
if (progress % 5000 == 0)
{
forker.OnProgress(this,
new ProgressEventArgs("Processing " + _catalogue,
new ProgressMeasurement(progress, ProgressType.Records), sw.Elapsed));
}
}
//final value
forker.OnProgress(this,
new ProgressEventArgs("Processing " + _catalogue,
new ProgressMeasurement(progress, ProgressType.Records), sw.Elapsed));
con.Close();
}
sw.Stop();
foreach (var state in byPivotRowStatesOverDataLoadRunId.Values)
{
state.CalculateFinalValues();
}
//now commit results
using (var con = dqeRepository.BeginNewTransactedConnection())
{
try
{
//mark down that we are beginning an evaluation on this the day of our lord etc...
Evaluation evaluation = new Evaluation(dqeRepository, _catalogue);
foreach (var state in byPivotRowStatesOverDataLoadRunId.Values)
{
state.CommitToDatabase(evaluation, _catalogue, con.Connection, con.Transaction);
}
if (_timePeriodicityField != null)
{
foreach (PeriodicityCubesOverTime periodicity in byPivotCategoryCubesOverTime.Values)
{
periodicity.CommitToDatabase(evaluation);
}
}
con.ManagedTransaction.CommitAndCloseConnection();
}
catch (Exception)
{
con.ManagedTransaction.AbandonAndCloseConnection();
throw;
}
}
forker.OnNotify(this,
new NotifyEventArgs(ProgressEventType.Information,
"CatalogueConstraintReport completed successfully and committed results to DQE server"));
}
catch (Exception e)
{
if (!(e is OperationCanceledException))
{
forker.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Fatal Crash", e));
}
else
{
forker.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "DQE Execution Cancelled", e));
}
}
finally
{
toDatabaseLogger.FinalizeTableLoadInfos();
}
}
public void TestBasicDataTableAnonymiser5(LoggerTestCase testCase)
{
//Create a names table that will go into the database
var dt = new DataTable();
dt.Columns.Add("Name");
dt.Rows.Add(new[] { "Thomas" });
dt.Rows.Add(new[] { "Wallace" });
dt.Rows.Add(new[] { "Frank" });
//upload the DataTable from memory into the database
var discoveredTable = GetCleanedServer(DatabaseType.MicrosoftSQLServer).CreateTable("ForbiddenNames", dt);
try
{
TableInfo tableInfo;
//import the persistent TableInfo reference
var importer = Import(discoveredTable, out tableInfo, out _);
//Create the test dataset chunks that will be anonymised
var dtStories1 = new DataTable();
dtStories1.Columns.Add("Story");
dtStories1.Rows.Add(new[] { "Thomas went to school regularly" }); //1st redact
dtStories1.Rows.Add(new[] { "It seems like Wallace went less regularly" }); //2nd redact
dtStories1.Rows.Add(new[] { "Mr Smitty was the teacher" });
var dtStories2 = new DataTable();
dtStories2.Columns.Add("Story");
dtStories2.Rows.Add(new[] { "Things were going so well" });
dtStories2.Rows.Add(new[] { "And then it all turned bad for Wallace" }); //3rd redact
var dtStories3 = new DataTable();
dtStories3.Columns.Add("Story");
dtStories3.Rows.Add(new[] { "There were things creeping in the dark" });
dtStories3.Rows.Add(new[] { "Surely Frank would know what to do. Frank was a genius" }); //4th redact
dtStories3.Rows.Add(new[] { "Mr Smitty was the teacher" });
//Create the anonymiser
var a = new BasicDataTableAnonymiser5();
//Tell it about the database table
a.NamesTable = tableInfo;
//Create a listener according to the test case
IDataLoadEventListener listener = null;
switch (testCase)
{
case LoggerTestCase.ToConsole:
listener = new ThrowImmediatelyDataLoadEventListener();
break;
case LoggerTestCase.ToMemory:
listener = new ToMemoryDataLoadEventListener(true);
break;
case LoggerTestCase.ToDatabase:
//get the default logging server
var logManager = CatalogueRepository.GetDefaultLogManager();
//create a new super task Anonymising Data Tables
logManager.CreateNewLoggingTaskIfNotExists("Anonymising Data Tables");
//setup a listener that goes to this logging database
listener = new ToLoggingDatabaseDataLoadEventListener(this, logManager, "Anonymising Data Tables", "Run on " + DateTime.Now);
break;
default:
throw new ArgumentOutOfRangeException("testCase");
}
//run the anonymisation
//process all 3 batches
a.ProcessPipelineData(dtStories1, listener, new GracefulCancellationToken());
a.ProcessPipelineData(dtStories2, listener, new GracefulCancellationToken());
a.ProcessPipelineData(dtStories3, listener, new GracefulCancellationToken());
//check the results
switch (testCase)
{
case LoggerTestCase.ToMemory:
Assert.AreEqual(4, ((ToMemoryDataLoadEventListener)listener).LastProgressRecieivedByTaskName["REDACTING Names"].Progress.Value);
break;
case LoggerTestCase.ToDatabase:
((ToLoggingDatabaseDataLoadEventListener)listener).FinalizeTableLoadInfos();
break;
}
}
finally
{
//finally drop the database table
discoveredTable.Drop();
}
}
