public void TestSerialPipelineExecution()
{
// set SetUp two engines, one with a locked cache progress/load schedule
// run the serial execution and ensure that only one engine had its 'ExecutePipeline' method called
var engine1 = new Mock <IDataFlowPipelineEngine>();
var engine2 = new Mock <IDataFlowPipelineEngine>();
var tokenSource = new GracefulCancellationTokenSource();
var listener = new ThrowImmediatelyDataLoadEventListener();
// set SetUp the engine map
var loadProgress1 = Mock.Of <ILoadProgress>();
var loadProgress2 = Mock.Of <ILoadProgress>();
// set SetUp the lock provider
var engineMap = new Dictionary <IDataFlowPipelineEngine, ILoadProgress>
{
{ engine1.Object, loadProgress1 },
{ engine2.Object, loadProgress2 }
};
// create the execution object
var pipelineExecutor = new SerialPipelineExecution();
// Act
pipelineExecutor.Execute(new [] { engine1.Object, engine2.Object }, tokenSource.Token, listener);
// engine1 should have been executed once
engine1.Verify(e => e.ExecutePipeline(It.IsAny <GracefulCancellationToken>()), Times.Once);
// engine2 should also have been run (locking isn't a thing anymore)
engine2.Verify(e => e.ExecutePipeline(It.IsAny <GracefulCancellationToken>()), Times.Once);
}
private void BtnRun_Clicked()
{
if (cancellation != null)
{
MessageBox.ErrorQuery("Already Running", "Pipeline is already running", "Ok");
return;
}
runner = new PipelineRunner(_useCase, _pipeline);
foreach (var l in additionals)
{
runner.AdditionalListeners.Add(l);
}
runner.PipelineExecutionFinishedsuccessfully += Runner_PipelineExecutionFinishedsuccessfully;
// clear old results
_results.Text = "";
_results.SelectedItem = 0;
Task.Run(() =>
{
try
{
cancellation = new GracefulCancellationTokenSource();
exitCode = runner.Run(activator.RepositoryLocator, this, new FromDataLoadEventListenerToCheckNotifier(this), cancellation.Token);
cancellation = null;
}
catch (Exception ex)
{
OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, ex.Message, ex));
cancellation = null;
}
});
}
private void btnExecute_Click(object sender, EventArgs e)
{
_cancellationTokenSource = new GracefulCancellationTokenSource();
IRunner runner;
try
{
var command = CommandGetter(CommandLineActivity.run);
runner = _factory.CreateRunner(Activator, command);
}
catch (Exception ex)
{
ragChecks.Fatal(ex);
return;
}
CurrentRunner = runner;
loadProgressUI1.Clear();
loadProgressUI1.ShowRunning(true);
int exitCode = 0;
_runningTask =
//run the data load in a Thread
Task.Factory.StartNew(() => { exitCode = Run(runner); });
_runningTask
//then on the main UI thread (after load completes with success/error
.ContinueWith((t) =>
{
//reset the system state because the execution has completed
ChecksPassed = false;
loadProgressUI1.ShowRunning(false);
if (exitCode != 0)
{
loadProgressUI1.SetFatal();
}
if (ExecutionFinished != null)
{
ExecutionFinished(this, new ExecutionEventArgs(exitCode));
}
//adjust the buttons accordingly
SetButtonStates();
}
, TaskScheduler.FromCurrentSynchronizationContext());
SetButtonStates();
}
public void LoadNotRequiredStopsPipelineGracefully()
{
var component = new NotRequiredComponent();
var pipeline = new SingleJobExecution(new List <IDataLoadComponent> {
component
});
var job = Mock.Of <IDataLoadJob>();
var jobTokenSource = new GracefulCancellationTokenSource();
pipeline.Run(job, jobTokenSource.Token);
}
public void ColumnDropper_NoMatchingColumnAtRuntime()
{
var dropper = new ColumnDropper
{
ColumnNameToDrop = "DoesNotExist"
};
var cts = new GracefulCancellationTokenSource();
var toProcess = new DataTable();
toProcess.Columns.Add("Column1");
var ex = Assert.Throws <InvalidOperationException>(() => dropper.ProcessPipelineData(toProcess, new ThrowImmediatelyDataLoadEventListener(), cts.Token));
Assert.IsTrue(ex.Message.Contains("does not exist in the supplied data table"));
}
public void ColumnDropper_Successful()
{
var dropper = new ColumnDropper
{
ColumnNameToDrop = "ToDrop"
};
var cts = new GracefulCancellationTokenSource();
var toProcess = new DataTable();
toProcess.Columns.Add("ToDrop");
var processed = dropper.ProcessPipelineData(toProcess, new ThrowImmediatelyDataLoadEventListener(), cts.Token);
Assert.AreEqual(0, processed.Columns.Count);
Assert.AreEqual(false, processed.Columns.Contains("ToDrop"));
}
public void ColumnRenamer_Successful()
{
var renamer = new ColumnRenamer
{
ColumnNameToFind = "ToFind",
ReplacementName = "ReplacementName"
};
var cts = new GracefulCancellationTokenSource();
var toProcess = new DataTable();
toProcess.Columns.Add("ToFind");
var processed = renamer.ProcessPipelineData(toProcess, new ThrowImmediatelyDataLoadEventListener(), cts.Token);
Assert.AreEqual(1, processed.Columns.Count);
Assert.AreEqual("ReplacementName", processed.Columns[0].ColumnName);
}
public void TestRoundRobinPipelineExecution()
{
// set SetUp two engines, one with a locked cache progress/load schedule
// run the serial execution and ensure that only one engine had its 'ExecutePipeline' method called
var engine1 = new Mock <IDataFlowPipelineEngine>();
var engine2 = new Mock <IDataFlowPipelineEngine>();
var tokenSource = new GracefulCancellationTokenSource();
var listener = new ThrowImmediatelyDataLoadEventListener();
// first time both engines return that they have more data, second time they are both complete
engine1.SetupSequence(engine => engine.ExecuteSinglePass(It.IsAny <GracefulCancellationToken>()))
.Returns(true)
.Returns(false)
.Throws <InvalidOperationException>();
engine2.SetupSequence(engine => engine.ExecuteSinglePass(It.IsAny <GracefulCancellationToken>()))
.Returns(true)
.Returns(false)
.Throws <InvalidOperationException>();
// set SetUp the engine map
var loadProgress1 = Mock.Of <ILoadProgress>();
var loadProgress2 = Mock.Of <ILoadProgress>();
// set SetUp the lock provider
var engineMap = new Dictionary <IDataFlowPipelineEngine, ILoadProgress>
{
{ engine1.Object, loadProgress1 },
{ engine2.Object, loadProgress2 }
};
// create the execution object
var pipelineExecutor = new RoundRobinPipelineExecution();
// Act
pipelineExecutor.Execute(new[] { engine1.Object, engine2.Object }, tokenSource.Token, listener);
// Assert
// engine1 should have been executed once
engine1.Verify();
// engine2 should not have been executed as it is locked
engine1.Verify();
}
/// <summary>
/// Blocking call which runs a set of DataFlowPipelineEngines according to the execution strategy whilst observing the PermissionWindow.
/// Stops the task and returns if the PermissionWindow closes.
/// </summary>
/// <param name="cancellationToken"></param>
/// <param name="cachingEngines"></param>
/// <returns></returns>
private RetrievalResult RunOnce(GracefulCancellationToken cancellationToken, List <IDataFlowPipelineEngine> cachingEngines)
{
// We will be spawning our own task which we want separate control of (to kill if we pass outside the permission window), so need our own cancellation token
var executionCancellationTokenSource = new GracefulCancellationTokenSource();
// We want to be able to stop the engine if we pass outside the permission window, however the execution strategy objects should not know about PermissionWindows
var executionTask = new Task(() =>
_pipelineEngineExecutionStrategy.Execute(cachingEngines, executionCancellationTokenSource.Token, _listener));
// Block waiting on task completion or signalling of the cancellation token
while (!executionTask.IsCompleted)
{
if (executionTask.Status == TaskStatus.Created)
{
executionTask.Start();
}
Task.Delay(1000).Wait();
// We need to handle stop and abort as we have used our own cancellation token with the child task
// If someone above us in the process chain has requested abort or cancel then use our own cancellation token to pass this info on to the child task
if (cancellationToken.IsAbortRequested)
{
// Wait nicely until the child task signals its abort token (by throwing?)
executionCancellationTokenSource.Abort();
_listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Have issued Abort request to the Pipeline Execution Task. Waiting for the task to exit..."));
try
{
executionTask.Wait();
}
catch (AggregateException)
{
}
return(RetrievalResult.Aborted);
}
if (cancellationToken.IsStopRequested)
{
// Wait nicely until the child task signals its stop token (by throwing?)
executionCancellationTokenSource.Stop();
_listener.OnNotify(this,
new NotifyEventArgs(ProgressEventType.Information,
"Have issued Stop request to the Pipeline Execution Task, however this may take some to complete as it will attempt to complete the current run through the pipeline."));
executionTask.Wait();
return(RetrievalResult.Stopped);
}
// Now can check to see if we are finished (we have passed outside the permission window)
if (_permissionWindow != null && !_permissionWindow.WithinPermissionWindow())
{
executionCancellationTokenSource.Abort();
_listener.OnNotify(this,
new NotifyEventArgs(ProgressEventType.Information,
"Now outside the PermissionWindow, have issued Abort request to the Pipeline Execution Task."));
executionTask.Wait();
return(RetrievalResult.NotPermitted);
}
}
if (executionTask.IsFaulted)
{
_listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Task faulted, information is in the attached exception.", executionTask.Exception));
throw new InvalidOperationException("Task faulted, see inner exception for details.", executionTask.Exception);
}
return(RetrievalResult.Complete);
}
