public integrated_performance_data_collection_testing()
{
_runner = new SpecRunner <GrammarSystem>();
_hierarchy = TestingContext.Hierarchy;
theResults = runSpec("sentence4");
}
public bool ShouldRetry(SpecResults results, Specification specification, SpecRunnerStatus status)
{
if (results.Counts.WasSuccessful())
{
return(false);
}
if (status == SpecRunnerStatus.Invalid)
{
return(false);
}
if (results.HadCriticalException)
{
return(false);
}
if (specification.Lifecycle == Lifecycle.Acceptance)
{
return(false);
}
return(specification.MaxRetries > (results.Attempts - 1) ||
Project.CurrentMaxRetries > (results.Attempts));
}
public void SetUp()
{
_runner = new SpecRunner <GrammarSystem>();
_hierarchy = TestingContext.Hierarchy;
theResults = runSpec("sentence4");
}
public void SpecHandled(SpecificationPlan plan, SpecResults specResults)
{
var record = _records[plan.Specification.id];
record.results = specResults;
record.specification = plan.Specification;
CompleteCheck();
}
public void SpecHandled(SpecExecutionRequest request, SpecResults results)
{
var message = $"{(++_number).ToString().PadLeft(_padding)}/{_total}). {request.Specification.path}";
var color = results.Counts.WasSuccessful() ? ConsoleColor.Green : ConsoleColor.Red;
ConsoleWriter.Write(color, message);
}
public void SpecHandled(SpecificationPlan plan, SpecResults specResults)
{
var record = _records[plan.Specification.id];
record.results = specResults;
record.specification = plan.Specification;
CompleteCheck();
}
public SpecExecutionCompleted(string id, SpecResults results, Specification data) : this()
{
if (results == null) throw new ArgumentNullException(nameof(results));
if (data == null) throw new ArgumentNullException(nameof(data));
Results = results;
Id = id;
Time = DateTime.Now.ToString("T");
Data = data;
}
protected override void theContextIs()
{
receiveRunSpec("embeds");
var node = findSpec("embeds");
theResults = new SpecResults {
Counts = new Counts(1, 0, 0, 0)
};
ClassUnderTest.SpecExecutionFinished(node, theResults);
}
public when_receiving_the_spec_execution_finished()
{
receiveRunSpec("embeds");
var node = findSpec("embeds");
theResults = new SpecResults {
Counts = new Counts(1, 0, 0, 0)
};
ClassUnderTest.SpecExecutionFinished(node, theResults);
}
public executing_when_spec_has_expired()
{
var specNode = new Specification()
{
id = Guid.NewGuid().ToString()
};
MockFor <ISpecExpiration>().IsExpired(Arg.Any <Specification>()).Returns(true);
theRequest = new SpecExecutionRequest(specNode, new NulloResultObserver());
theResults = ClassUnderTest.Execute(theRequest, MockFor <IConsumingQueue>());
}
protected override void beforeEach()
{
var specNode = new Specification()
{
id = Guid.NewGuid().ToString()
};
MockFor <ISpecExpiration>().Stub(x => x.IsExpired(Arg <Specification> .Is.Anything)).Return(true);
theRequest = new SpecExecutionRequest(specNode, new NulloResultObserver());
theResults = ClassUnderTest.Execute(theRequest, MockFor <IConsumingQueue>());
}
public void AfterRunning(SpecExecutionRequest request, SpecResults results, IConsumingQueue queue, SpecRunnerStatus status)
{
if (ShouldRetry(results, request.Specification, status))
{
_resultObserver.SpecRequeued(request);
queue.Enqueue(request);
}
else
{
_resultObserver.SpecHandled(request, results);
}
}
public void AfterRunning(SpecExecutionRequest request, SpecResults results, IConsumingQueue queue, SpecRunnerStatus status)
{
if (ShouldRetry(results, request.Specification, status))
{
_resultObserver.SpecRequeued(request);
queue.Enqueue(request);
}
else
{
_resultObserver.SpecHandled(request, results);
}
}
protected override void beforeEach()
{
ClassUnderTest.Status = SpecRunnerStatus.Invalid;
var specNode = new Specification()
{
id = Guid.NewGuid().ToString()
};
theRequest = new SpecExecutionRequest(specNode, new NulloResultObserver());
theResults = ClassUnderTest.Execute(theRequest, MockFor <IConsumingQueue>());
}
public bool ShouldRetry(SpecResults results, Specification specification, SpecRunnerStatus status)
{
if (results.Counts.WasSuccessful()) return false;
if (status == SpecRunnerStatus.Invalid) return false;
if (results.HadCriticalException) return false;
if (specification.Lifecycle == Lifecycle.Acceptance) return false;
return specification.MaxRetries > (results.Attempts - 1) ||
Project.CurrentMaxRetries > (results.Attempts);
}
public void load_specification_returns_the_specifiction_and_result_if_it_exists()
{
var theResults = new SpecResults();
var completed = new SpecExecutionCompleted("sentence4", theResults, new Specification());
ClassUnderTest.Receive(completed);
var message = ClassUnderTest.LoadSpecification("sentence4");
message.id.ShouldBe("sentence4");
message.data.ShouldBeOfType <Specification>().id.ShouldBe("sentence4");
message.results.ShouldContain(completed);
}
public executing_when_the_runner_is_invalid()
{
ClassUnderTest.Status = SpecRunnerStatus.Invalid;
var specNode = new Specification()
{
id = Guid.NewGuid().ToString()
};
theRequest = new SpecExecutionRequest(specNode, new NulloResultObserver());
theResults = ClassUnderTest.Execute(theRequest, MockFor <IConsumingQueue>());
}
public void build_for_an_aborted_spec_run()
{
var results = SpecResults.ForAbortedRun("foo");
results.Attempts.ShouldBe(0);
results.Counts.ShouldEqual(0, 0, 1, 0);
results.Performance.Length.ShouldBe(0);
results.Results.Length.ShouldBe(1);
results.Results.Single().id.ShouldBe("foo");
results.WasAborted.ShouldBe(true);
}
public void SpecExecutionFinished(Specification specification, SpecResults results)
{
var outstanding = _outstanding.FirstOrDefault(x => Equals(x.Specification, specification));
if (outstanding == null)
{
return;
}
_outstanding.Remove(outstanding);
_observer.SendToClient(new SpecExecutionCompleted(specification.id, results, specification));
SendQueueState();
}
public BatchExecutionModeTester()
{
Project.CurrentProject = new Project {
MaxRetries = 0
};
theSpecification = new Specification();
theResults = new SpecResults
{
HadCriticalException = false,
Counts = new Counts(0, 1, 0, 0)
};
theStatus = SpecRunnerStatus.Valid;
}
public void finishing_a_spec()
{
var action = Substitute.For <IResultObserver>();
var request = new SpecExecutionRequest(theSpec, action);
request.CreatePlan(TestingContext.Library);
request.Plan.Attempts = 3;
var results = new SpecResults();
request.SpecExecutionFinished(results);
action.Received().SpecExecutionFinished(theSpec, results);
}
protected override void beforeEach()
{
Project.CurrentProject = new Project {
MaxRetries = 0
};
theSpecification = new Specification();
theResults = new SpecResults
{
HadCriticalException = false,
Counts = new Counts(0, 1, 0, 0)
};
theStatus = SpecRunnerStatus.Valid;
}
public SpecExecutionCompleted(string id, SpecResults results, Specification data) : this()
{
if (results == null)
{
throw new ArgumentNullException("results");
}
if (data == null)
{
throw new ArgumentNullException("data");
}
Results = results;
Id = id;
Time = DateTime.Now.ToString("T");
Data = data;
}
public void finishing_a_spec()
{
var action = MockRepository.GenerateMock <IResultObserver>();
var request = new SpecExecutionRequest(theSpec, action);
request.ReadXml();
request.CreatePlan(TestingContext.Library);
request.Plan.Attempts = 3;
var results = new SpecResults();
request.SpecExecutionFinished(results);
action.AssertWasCalled(x => x.SpecExecutionFinished(theSpec, results));
}
public SpecResults Execute(SpecExecutionRequest request, IConsumingQueue queue)
{
_mode.BeforeRunning(request);
if (Status == SpecRunnerStatus.Invalid)
{
var abortedResults = SpecResults.ForAbortedRun(request.Id);
_mode.AfterRunning(request, abortedResults, queue, Status);
return(abortedResults);
}
if (_specExpiration.IsExpired(request.Specification))
{
var expiredResults = SpecResults.ForExpiredRun(request.Id);
_mode.AfterRunning(request, expiredResults, queue, Status);
return(expiredResults);
}
request.Plan.Attempts++;
var timings = request.StartNewTimings();
SpecResults results = null;
try
{
_current = new ExecutionRun(_system, timings, request, _stopConditions, _mode);
results = _current.Execute();
}
catch (Exception ex) // Any exception that bubbles up is telling us that the runner is invalid
{
MarkAsInvalid(ex);
results = buildResultsForContextCreationFailure(request, ex, timings);
}
finally
{
_mode.AfterRunning(request, results, queue, Status);
timings.Dispose();
if (_current != null)
{
_current.SafeDispose();
}
}
return(results);
}
public void SpecHandled(SpecExecutionRequest request, SpecResults results)
{
var name = request.Specification.name.Escape();
var resultText = results.Counts.ToString();
if (results.Counts.WasSuccessful())
{
appVeyorAddTest(name, request.Specification.Filename, "Passed", results.Duration, stdOut: resultText);
}
else if (request.Specification.Lifecycle == Lifecycle.Acceptance)
{
appVeyorAddTest(name, request.Specification.Filename, "Ignored", results.Duration, stdOut: "Acceptance test failed: " + resultText);
}
else
{
appVeyorAddTest(name, request.Specification.Filename, "Failed", results.Duration, stdOut: resultText);
}
}
public void finishing_a_spec_finishes_the_completion()
{
var action = Substitute.For <IResultObserver>();
var request = new SpecExecutionRequest(theSpec, action);
request.CreatePlan(TestingContext.Library);
request.Plan.Attempts = 3;
var results = new SpecResults();
request.Completion.IsCompleted.ShouldBeFalse();
request.SpecExecutionFinished(results);
request.Completion.IsCompleted.ShouldBeTrue();
request.Completion.Result.ShouldBe(results);
}
public SpecResults Execute(SpecExecutionRequest request, IConsumingQueue queue)
{
_mode.BeforeRunning(request);
if (Status == SpecRunnerStatus.Invalid)
{
var abortedResults = SpecResults.ForAbortedRun(request.Id);
_mode.AfterRunning(request, abortedResults, queue, Status);
return(abortedResults);
}
request.Plan.Attempts++;
var timings = request.StartNewTimings();
SpecResults results = null;
try
{
Current = request.Mode == ExecutionMode.normal
? new SpecExecution(request, _stopConditions, _mode.BuildLogger())
: new StepthroughExecution(request, _stopConditions, _mode.Observer(), _observer);
_observer.SpecStarted(request);
results = Current.Execute(_system, timings);
}
catch (Exception ex) // Any exception that bubbles up is telling us that the runner is invalid
{
MarkAsInvalid(ex);
results = buildResultsForContextCreationFailure(request, ex, timings);
}
finally
{
_mode.AfterRunning(request, results, queue, Status);
timings.Dispose();
}
return(results);
}
public void SpecHandled(SpecExecutionRequest request, SpecResults results)
{
var name = request.Specification.name.Escape();
var resultText = results.Counts.ToString().Escape();
if (results.Counts.WasSuccessful())
{
Console.WriteLine("##teamcity[testFinished name='{0}' message='{1}']", name, resultText);
}
else if (request.Specification.Lifecycle == Lifecycle.Acceptance)
{
Console.WriteLine("##teamcity[testIgnored name='{0}' message='{1}']", name,
"Acceptance test failed: " + resultText);
}
else
{
Console.WriteLine("##teamcity[testFailed name='{0}' details='{1}']", name,
resultText);
}
}
public executing_a_spec_when_context_creation_blows_up()
{
var specNode = new Specification()
{
id = Guid.NewGuid().ToString()
};
theRequest = new SpecExecutionRequest(specNode, new NulloResultObserver());
theRequest.CreatePlan(TestingContext.Library);
theQueue = MockFor <IConsumingQueue>();
theException = new DivideByZeroException("Wrong!");
MockFor <ISystem>().CreateContext().Throws(theException);
ClassUnderTest.Status.ShouldBe(SpecRunnerStatus.Valid);
theResults = ClassUnderTest.Execute(theRequest, theQueue);
}
protected override void beforeEach()
{
var specNode = new Specification()
{
id = Guid.NewGuid().ToString()
};
theRequest = new SpecExecutionRequest(specNode, new NulloResultObserver());
theRequest.CreatePlan(TestingContext.Library);
theQueue = MockFor <IConsumingQueue>();
theException = new DivideByZeroException("Wrong!");
MockFor <ISystem>().Stub(x => x.CreateContext()).Throw(theException);
ClassUnderTest.Status.ShouldBe(SpecRunnerStatus.Valid);
theResults = ClassUnderTest.Execute(theRequest, theQueue);
}
public void SpecHandled(SpecExecutionRequest request, SpecResults results)
{
_inner.SpecHandled(request, results);
var name = request.Specification.name.Escape();
var resultText = results.Counts.ToString().Escape();
if (results.Counts.WasSuccessful())
{
Console.WriteLine("##teamcity[testFinished name='{0}' message='{1}']", name, resultText);
}
else if (request.Specification.Lifecycle == Lifecycle.Acceptance)
{
Console.WriteLine("##teamcity[testIgnored name='{0}' message='{1}']", name,
"Acceptance test failed: " + resultText);
}
else
{
Console.WriteLine("##teamcity[testFailed name='{0}' details='{1}']", name,
resultText);
}
}
public void SpecExecutionFinished(Specification specification, SpecResults results)
{
var completed = new SpecExecutionCompleted(specification.id, results, specification);
_completed.Add(specification.id, completed);
}
public void AfterRunning(SpecExecutionRequest request, SpecResults results, IConsumingQueue queue, SpecRunnerStatus status)
{
}
public void SpecExecutionFinished(SpecResults results)
{
Observer.SpecExecutionFinished(Specification, results);
}
public void AfterRunning(SpecExecutionRequest request, SpecResults results, IConsumingQueue queue, SpecRunnerStatus status)
{
}
public void SpecExecutionFinished(SpecResults results)
{
_observer.SpecExecutionFinished(Specification, results);
}
