/// <summary>
/// Updates the agent.
/// </summary>
/// <param name="agent">The agent.</param>
public void UpdateAgent(AgentMetadata agent)
{
try
{
var agentVersion = agent.Version;
var currentVersion = versionProvider.GetVersion();
if (agentVersion < currentVersion)
{
var zippedVersionFolder = updateSource.GetZippedVersionFolder();
if (zippedVersionFolder != null)
{
agents.MarkAsUpdating(agent);
new Action(() =>
{
try
{
var testRunnerAgent =
connectionProvider.GetConnection<IRemoteAppPart>(agent.RemotePartAddress);
testRunnerAgent.ReceiveUpdatePackage(new UpdatePackage
{
IsAvailable = true,
Version = currentVersion,
UpdateZipStream =
zippedVersionFolder
});
agents.MarkAsUpdated(agent);
}
catch(CommunicationException ex)
{
agents.MarkAsDisconnected(agent);
log.Error("Error while updating agent", ex);
throw;
}
catch(Exception ex)
{
agents.MarkAsFailure(agent);
log.Error("Error while updating agent", ex);
throw;
}
}).BeginInvoke(null, null);
}
}
}
catch (CommunicationException ex)
{
log.Error("Error while trying to check version or apply updates", ex);
agents.MarkAsFailure(agent);
}
}
private void AddToCollection(AgentMetadata agent)
{
using (agents.Lock())
{
// ReSharper disable AccessToModifiedClosure
var agentWithSameNameSameAddress = agents.GetAgent(a => string.Equals(a.Name, agent.Name)
&& a.Address.Equals(agent.Address));
var agentWithSameName = agents.GetAgentByName(agent.Name);
var agentWithSameAddress = agents.GetAgentByAddress(agent.Address);
// ReSharper restore AccessToModifiedClosure
if (agentWithSameNameSameAddress != null && agentWithSameNameSameAddress.Status == AgentState.Updated)
{
if (agentWithSameNameSameAddress.Version < agent.Version) // agent was updated and restarted
StopAgentPinging(agentWithSameNameSameAddress);
else
return; // State was set to updated, but the agent has not been restarted yet
}
else if (agentWithSameNameSameAddress != null && agentWithSameNameSameAddress.Status != AgentState.Disconnected)
return;
if (agentWithSameNameSameAddress != null) //agent is disconnected here
{
agentWithSameNameSameAddress.Version = agent.Version;
agent = agentWithSameNameSameAddress; // replace, so it can be later connected
}
else
{
if (agentWithSameName != null)
{
StopAgentPinging(agentWithSameName);
agents.MarkAsDisconnected(agentWithSameName);
}
if (agentWithSameAddress != null)
{
StopAgentPinging(agentWithSameAddress);
agents.MarkAsDisconnected(agentWithSameAddress);
}
}
agents.Connect(agent);
StartAgentPinging(agent);
}
}
public void ChildrenAreAddedAsSeparateUnits()
{
unitsFixture.Add("As.Nam1.Nam2.Nam3.Nam4.S1",
new[]
{
"tc1",
"tc2",
"tc3"
});
var units = unitsFixture.Build();
resultsfactory.Initialize(units);
resultsfactory.Execute(units[0].Children[0], r => r.Failure("Unit 1 failed", "Error stack trace 1"));
resultsfactory.Execute(units[0].Children[2], r => r.Failure("Unit 3 failed", "Error stack trace 3"));
var result = resultsfactory.Build();
testRun.Parameters.SpecialHandlings.Add(
new TestRunFailureSpecialHandling
{
FailureMessage = "Unit 1",
FailureMessageType = MatchType.ContainsText,
RetryCount = 2
});
testRun.Parameters.SpecialHandlings.Add(
new TestRunFailureSpecialHandling
{
FailureMessage = "^U.+led$",
FailureMessageType = MatchType.Regex,
RetryCount = 2
});
var agent = new AgentMetadata(new EndpointAddress("net.tcp://test/"));
reprocessor.AddForReprocessingIfRequired(units[0], result, agent);
collectionMock.Received(1)
.Add(Arg.Is<TestUnitWithMetadata>(v => v.FullName.Equals("As.Nam1.Nam2.Nam3.Nam4.S1.tc1")));
collectionMock.Received(1)
.Add(Arg.Is<TestUnitWithMetadata>(v => v.FullName.Equals("As.Nam1.Nam2.Nam3.Nam4.S1.tc3")));
}
public void FailedResultsIsProcessWhenMatchingInstructionsGiven()
{
unitsFixture.Add("As.Nam1.Nam2.Nam3.Nam4.S1",
new[]
{
"tc1",
"tc2",
"tc3"
});
unitsFixture.Add("As.Nam1.Nam2.Nam3.Nam4.S2",
new[]
{
"tc1",
"tc2",
"tc3"
});
var units = unitsFixture.Build();
resultsfactory.Initialize(units);
resultsfactory.Execute(units[0], r => r.Error(resultsfactory.GetInitilalizedException<NotImplementedException>()));
resultsfactory.Execute(units[1], r => r.Failure("Unit 1 failed", "Error stack trace"));
var result = resultsfactory.Build();
testRun.Parameters.SpecialHandlings.Add(new TestRunFailureSpecialHandling
{
FailureMessage = "Unit 1",
FailureMessageType = MatchType.ContainsText,
RetryCount = 2
});
testRun.Parameters.SpecialHandlings.Add(new TestRunFailureSpecialHandling
{
FailureStackTrace = resultsfactory.GetType().Namespace,
FailureStackTraceType = MatchType.ContainsText,
RetryCount = 2
});
var agent = new AgentMetadata(new EndpointAddress("net.tcp://test/"));
reprocessor.AddForReprocessingIfRequired(units[0], result, agent);
reprocessor.AddForReprocessingIfRequired(units[1], result, agent);
collectionMock
.Received(1)
.Add(Arg.Is<TestUnitWithMetadata>(v => v.FullName.Equals("As.Nam1.Nam2.Nam3.Nam4.S1")));
collectionMock.Received(1)
.Add(Arg.Is<TestUnitWithMetadata>(v => v.FullName.Equals("As.Nam1.Nam2.Nam3.Nam4.S2")));
}
public void FailedOrErroredResultIsNotReprocessedWhenNoHandlingInstructionsGiven()
{
unitsFixture.Add("As.Nam1.Nam2.Nam3.Nam4.S1",
new[]
{
"tc1",
"tc2",
"tc3"
});
unitsFixture.Add("As.Nam1.Nam2.Nam3.Nam4.S2",
new[]
{
"tc1",
"tc2",
"tc3"
});
var units = unitsFixture.Build();
resultsfactory.Initialize(units);
resultsfactory.Execute(units[0], r => r.Error(resultsfactory.GetInitilalizedException<NotImplementedException>()));
resultsfactory.Execute(units[1], r => r.Failure("Error message", "Error stack trace"));
var result = resultsfactory.Build();
var agent = new AgentMetadata(new EndpointAddress("net.tcp://test/"));
reprocessor.AddForReprocessingIfRequired(units[0], result, agent);
reprocessor.AddForReprocessingIfRequired(units[1], result, agent);
collectionMock.DidNotReceiveWithAnyArgs().Add(Arg.Any<TestUnitWithMetadata>());
}
// Should be started in a separate thread
private void Run(TestUnitWithMetadata test, AgentMetadata agent, DistributedConfigurationSubstitutions configurationSubstitutions)
{
log.BeginActivity(string.Format("[{0}] Started test {1}", test.Test.Run, test.Test.Info.TestName.FullName));
var request = requests.GetBy(test.Test.Run);
if (request != null)
request.Status = TestRunRequestStatus.Pending;
try
{
log.BeginActivity(string.Format("Connecting to agent [{0}]...", agent));
var testRunnerAgent = connectionProvider.GetConnection<IAgent>(agent.Address);
log.BeginActivity(string.Format("Connected to agent [{0}]", agent));
log.BeginActivity(string.Format("Checking project existence ('{0}') on agent {1}", test.Test.Run, agent));
if (!testRunnerAgent.HasProject(test.Test.Run))
{
log.EndActivity(string.Format("Project '{0}' doesn't exist on agent {1}", test.Test.Run, agent));
log.BeginActivity(string.Format("Sending project ('{0}') to agent {1}", test.Test.Run, agent));
using (Stream project = projects.GetStreamToPacked(test.Test.Run) ?? new MemoryStream(1))
{
testRunnerAgent.ReceiveProject(new ProjectMessage
{
TestRun = test.Test.Run,
Project = project
});
}
log.EndActivity(string.Format("Sent project ('{0}') to agent {1}", test.Test.Run, agent));
}
else
log.EndActivity(string.Format("Project '{0}' exist on agent {1}", test.Test.Run, agent));
var reprocessedCount = request.Statistics.ReprocessedCount;
log.BeginActivity(string.Format("[{3}/{4}]: Running {0} on {1} with variables ({2})...", test.Test.UniqueTestId, agent, configurationSubstitutions,
request.Statistics.GetCountAndIncrement(), reprocessedCount == 0
? request.Statistics.Total.ToString(CultureInfo.InvariantCulture)
: string.Format("{2}({0}+{1})", request.Statistics.Total, reprocessedCount, request.Statistics.Total + reprocessedCount)));
TestResult result = testRunnerAgent.RunTests(test.Test, configurationSubstitutions);
log.EndActivity(string.Format("Finished running {0} on {1}...", test.Test.UniqueTestId, agent));
if (result == null)
throw new FaultException("Result is not available");
ProcessResult(test, agent, result);
}
catch (FaultException ex)
{
log.Error(string.Format("Exception while running test {0} on {1}", test.Test.UniqueTestId, agent), ex);
agents.MarkAsFailure(agent);
tests.Add(test);
}
catch (CommunicationException ex)
{
log.Error(string.Format("Exception in communication while running test {0} on {1}", test.Test.UniqueTestId, agent), ex);
agents.MarkAsDisconnected(agent);
tests.Add(test);
}
catch(Exception ex)
{
log.Error(string.Format("Something bad and unhandled happened while running test {0} on {1}", test.Test.UniqueTestId, agent), ex);
}
}
private void ChangeStatus(AgentMetadata agent, AgentState agentState)
{
if (agent == null)
return;
var oldStatus = agent.Status;
((IAgentStateSetter)agent).Status = agentState;
if (oldStatus != agentState)
{
log.Debug(string.Format("Agent '{0}' state was changed from {1} to {2}", agent.Address, oldStatus, agentState));
}
if (oldStatus != AgentState.Ready && agentState == AgentState.Ready)
ReadyAgentAppeared.SafeInvoke(this, EventArgs.Empty);
else if (!oldStatus.IsOneOf(new[]{AgentState.Disconnected, AgentState.Error})
&& agentState.IsOneOf(new[]{AgentState.Disconnected, AgentState.Error}))
{
ClientDisconnectedOrFailed.SafeInvoke(this, EventArgs.Empty);
}
}
/// <summary>
/// Marks as updated.
/// </summary>
/// <param name="agent">The agent.</param>
public void MarkAsUpdated(AgentMetadata agent)
{
ChangeStatus(agent, AgentState.Updated);
}
/// <summary>
/// Marks as failure.
/// </summary>
/// <param name="agent">The agent.</param>
public void MarkAsFailure(AgentMetadata agent)
{
lock (syncObject)
{
ChangeStatus(agent, AgentState.Error);
}
}
private void AddTestForReprocessing(TestUnitWithMetadata test, AgentMetadata agent)
{
test.AttachedData.MarkAgentAs(agent, SchedulingHint.NotRecommended);
collection.Add(test);
}
private void StopAgentPinging(AgentMetadata agent)
{
Timer timer;
if (timers.TryRemove(agent, out timer))
{
timer.Change(Timeout.Infinite, Timeout.Infinite);
timer.Dispose();
}
}
private void StartAgentPinging(AgentMetadata agent)
{
var timer = timers.GetOrAdd(agent, key => new Timer(OnPingTimerTick, key, Timeout.Infinite, Timeout.Infinite));
lock (timer)
{
timer.Change(options.PingIntervalInMiliseconds, Timeout.Infinite);
}
}
private void OnPingTimerTick(object state)
{
// This method is run in a separate thread.
// There is no need to try to make it asynchronous.
var agent = state as AgentMetadata;
Timer timer;
if (!timers.TryGetValue(agent, out timer))
return;
if (agent == null || (!agent.Status.IsOneOf(new[] { AgentState.Ready, AgentState.New, AgentState.Error })))
{
ReschedulePinging(timer);
return;
}
try
{
var agentRemote = connectionProvider.GetConnection<IRemoteAppPart>(agent.RemotePartAddress);
var result = agentRemote.Ping(TimeSpan.FromMilliseconds(options.PingIntervalInMiliseconds));
agent.Version = result.Version;
if (!string.Equals(agent.Name, result.Name))
{
agents.MarkAsDisconnected(agent);
var newAgent = new AgentMetadata(agent.Address)
{
Name = result.Name,
Version = result.Version
};
agents.Connect(newAgent);
StartAgentPinging(newAgent);
}
agentUpdater.UpdateAgent(agent);
ReschedulePinging(timer);
}
catch (CommunicationException)
{
// remove agent from tracking
// mark as disconnected
StopAgentPinging(agent);
agents.MarkAsDisconnected(agent);
}
catch // in case of any other exception, we assume, that the agent errored out.
{
// remove agent from tracking
// mark as errored out
StopAgentPinging(agent);
agents.MarkAsFailure(agent);
}
}
private void OnAgentConnected(object sender, EventArgs<EndpointAddress> e)
{
var agent = new AgentMetadata(e.Data);
using (agents.Lock())
{
var savedAgent = agents.GetAgentByAddress(agent.Address);
if (savedAgent != null && !savedAgent.Status.IsOneOf(AgentState.Disconnected, AgentState.Updated))
return;
}
var action =
new Action(
() =>
{
try
{
LockCallToAddress(agent.Address);
log.Debug(string.Format("Initially pinging {0}", agent.Address));
var remoteAppPart = connectionProvider.GetConnection<IRemoteAppPart>(agent.RemotePartAddress);
var result = remoteAppPart.Ping(TimeSpan.FromMilliseconds(options.PingIntervalInMiliseconds));
log.Debug(string.Format("Successfully initially pinged {0} at {1}", result.Name, agent.Address));
agent.Name = result.Name;
agent.Version = result.Version;
AddToCollection(agent);
}
finally
{
UnlockCallToAddress(agent.Address);
}
});
action.BeginInvoke(ar =>
{
try
{
action.EndInvoke(ar);
}
catch(Exception ex)
{
// this should be traced as error
log.Debug(string.Format("An error occurred, while initially pinging and adding to collection the endpoint ({0}): {1}", agent.Address, ex.Message));
}
}, null);
}
public void ParentReprocessingIsPerformedFixedAmountOfTimes2()
{
unitsFixture.Add("As.Nam1.Nam2.Nam3.Nam4.S1",
new[]
{
"tc1",
"tc2",
"tc3"
});
var units = unitsFixture.Build();
resultsfactory.Initialize(units);
resultsfactory.Execute(units[0], r => r.Failure("U failed", "Error for suite"));
resultsfactory.Execute(units[0].Children[0], r => r.Ignore("Unit 1 failed", "Error stack trace 1"));
resultsfactory.Execute(units[0].Children[1], r => r.Success());
resultsfactory.Execute(units[0].Children[2], r => r.Failure("Unit 3 failed", "Error stack trace 3"));
var result = resultsfactory.Build();
testRun.Parameters.SpecialHandlings.Add(
new TestRunFailureSpecialHandling
{
FailureMessage = "^U.+led$",
FailureMessageType = MatchType.Regex,
RetryCount = 2
});
var agent = new AgentMetadata(new EndpointAddress("net.tcp://test/"));
reprocessor.AddForReprocessingIfRequired(units[0], result, agent);
for (int i = 0; i < 20; i++)
{
reprocessor.AddForReprocessingIfRequired(units[0], result, agent);
}
collectionMock.Received(2)
.Add(Arg.Is<TestUnitWithMetadata>(v => v.FullName.Equals("As.Nam1.Nam2.Nam3.Nam4.S1")));
collectionMock.DidNotReceive()
.Add(Arg.Is<TestUnitWithMetadata>(v => !v.FullName.Equals("As.Nam1.Nam2.Nam3.Nam4.S1")));
}
/// <summary>
/// Disconnects the specified agent.
/// </summary>
/// <param name="agent">The agent.</param>
public void MarkAsDisconnected(AgentMetadata agent)
{
lock (syncObject)
{
ChangeStatus(agent, AgentState.Disconnected);
}
}
/// <summary>
/// Adds for reprocessing if required.
/// </summary>
/// <param name="test">The test.</param>
/// <param name="result">The result.</param>
/// <param name="agent"> </param>
public void AddForReprocessingIfRequired(TestUnitWithMetadata test, TestResult result, AgentMetadata agent)
{
var request = requests != null ? requests.GetBy(test.Test.Run) : null;
Action registerReprocessing = () => { if (request != null) request.Statistics.RegisterReprocessing(); };
if (result == null)
{
test.AttachedData.NullReprocessingCount++;
collection.Add(test);
log.Info(string.Format("REPROCESSING (as null): '{0}' was added for reprocessing. [{1}]",
test.FullName, test.Test.Run));
registerReprocessing();
return;
}
if (!test.Test.Info.IsSuite)
{
var childResult = result.FindDescedant(d => d.FullName.Equals(test.FullName));
if (childResult != null
&& ProcessResult(childResult, test.Test.Run, test) == ReprocessingVerdict.PerformReprocessing)
{
AddTestForReprocessing(test, agent);
registerReprocessing();
}
return;
}
foreach (var suiteResult in result.FindBottomLevelTestSuites())
{
if (test.FullName.Equals(suiteResult.FullName))
{
var reprocessingVerdict = ProcessResult(suiteResult, test.Test.Run, test);
if (reprocessingVerdict == ReprocessingVerdict.MaximumCountWasReached)
return;
if (reprocessingVerdict == ReprocessingVerdict.PerformReprocessing)
{
AddTestForReprocessing(test, agent);
registerReprocessing();
continue;
}
}
var childrenForReprocessing = new List<Tuple<TestResult, TestUnitWithMetadata>>();
foreach (TestResult childResult in suiteResult.Results)
{
var childTest = test.Children.FirstOrDefault(ct => ct.FullName.Equals(childResult.FullName));
if (ProcessResult(childResult, test.Test.Run, childTest,
() =>
{
childTest = new TestUnitWithMetadata(test.Test.Run, childResult.Test, test.Test.AssemblyName);
test.Children.Add(childTest);
return childTest;
})==ReprocessingVerdict.PerformReprocessing)
childrenForReprocessing.Add(new Tuple<TestResult, TestUnitWithMetadata>(childResult, childTest));
}
if (childrenForReprocessing.Count > 0 && childrenForReprocessing.Count == suiteResult.Results.Count)
{
AddTestForReprocessing(test, agent);
registerReprocessing();
}
else
{
foreach (var childForReprocessing in childrenForReprocessing)
{
AddTestForReprocessing(childForReprocessing.Item2, agent);
registerReprocessing();
}
}
}
}
/// <summary>
/// Marks as ready.
/// </summary>
/// <param name="agent">The agent.</param>
public void MarkAsReady(AgentMetadata agent)
{
lock (syncObject)
{
ChangeStatus(agent, AgentState.Ready);
}
}
private DistributedConfigurationSubstitutions GetConfigurationValues(DistributedConfigurationSetup configurationSetup,
AgentMetadata agentToRun,
TestUnitWithMetadata testToRun)
{
//BUG: add dependency on test run!
return configurations.GetOrAdd(agentToRun.Name,
key =>
{
var distributedConfigurationValues = new DistributedConfigurationSubstitutions();
foreach (var variable in configurationSetup.Variables)
{
distributedConfigurationValues.Variables.Add(
new DistributedConfigurationVariablesValue(variable.Name, variable.GetNextValue()));
}
return distributedConfigurationValues;
});
}
/// <summary>
/// Marks the agent as updating.
/// </summary>
/// <param name="agent">The agent.</param>
public void MarkAsUpdating(AgentMetadata agent)
{
lock (syncObject)
{
ChangeStatus(agent, AgentState.Updating);
}
}
/// <summary>
/// Marks the agent as.
/// </summary>
/// <param name="agent">The agent.</param>
/// <param name="schedulingHint">The scheduling hint.</param>
public void MarkAgentAs(AgentMetadata agent, SchedulingHint schedulingHint)
{
}
/// <summary>
/// Connects the specified agent metadata.
/// </summary>
/// <param name="agent">The agent metadata.</param>
public void Connect(AgentMetadata agent)
{
lock (syncObject)
{
ChangeStatus(agent, AgentState.Ready);
if (!allAgents.Contains(agent))
allAgents.Add(agent);
}
}
private void ProcessResult(TestUnitWithMetadata test, AgentMetadata agent, TestResult result)
{
bool isRequestCompleted;
using (agents.Lock())
{
lock (tests.SyncObject)
{
AddResultsToTestUnitMetadata(test, result);
tests.MarkCompleted(test);
agents.MarkAsReady(agent);
reprocessor.AddForReprocessingIfRequired(test, result, agent);
isRequestCompleted = !tests.IsAnyAvailableFor(test.Test.Run);
}
}
results.Add(result, test.Test.Run);
var request = requests.GetBy(test.Test.Run);
if (request != null)
request.PipeToClient.Publish(result.DeepClone().SetFinal(false));
if (isRequestCompleted)
ProcessCompletedTestRun(test.Test.Run);
}