public async void CreatesSession()
{
using (TestHostContext tc = CreateTestContext())
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = tc.GetTrace();
// Arrange.
var expectedSession = new TaskAgentSession();
_runnerServer
.Setup(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is <TaskAgentSession>(y => y != null),
tokenSource.Token))
.Returns(Task.FromResult(expectedSession));
_credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
// Act.
MessageListener listener = new MessageListener();
listener.Initialize(tc);
bool result = await listener.CreateSessionAsync(tokenSource.Token);
trace.Info("result: {0}", result);
// Assert.
Assert.True(result);
_runnerServer
.Verify(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is <TaskAgentSession>(y => y != null),
tokenSource.Token), Times.Once());
}
}
public async void DeleteSession()
{
using (TestHostContext tc = CreateTestContext())
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = tc.GetTrace();
// Arrange.
var expectedSession = new TaskAgentSession();
PropertyInfo sessionIdProperty = expectedSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(sessionIdProperty);
sessionIdProperty.SetValue(expectedSession, Guid.NewGuid());
_runnerServer
.Setup(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is <TaskAgentSession>(y => y != null),
tokenSource.Token))
.Returns(Task.FromResult(expectedSession));
_credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
_store.Setup(x => x.GetCredentials()).Returns(new CredentialData()
{
Scheme = Constants.Configuration.OAuthAccessToken
});
_store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
// Act.
MessageListener listener = new MessageListener();
listener.Initialize(tc);
bool result = await listener.CreateSessionAsync(tokenSource.Token);
Assert.True(result);
_runnerServer
.Setup(x => x.DeleteAgentSessionAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask);
await listener.DeleteSessionAsync();
//Assert
_runnerServer
.Verify(x => x.DeleteAgentSessionAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny <CancellationToken>()), Times.Once());
}
}
public async void CreateSessionWithOriginalCredential()
{
using (TestHostContext tc = CreateTestContext())
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = tc.GetTrace();
// Arrange.
var expectedSession = new TaskAgentSession();
_runnerServer
.Setup(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is <TaskAgentSession>(y => y != null),
tokenSource.Token))
.Returns(Task.FromResult(expectedSession));
_credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
var originalCred = new CredentialData()
{
Scheme = Constants.Configuration.OAuth
};
originalCred.Data["authorizationUrl"] = "https://s.server";
originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897";
_store.Setup(x => x.GetCredentials()).Returns(originalCred);
_store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
// Act.
MessageListener listener = new MessageListener();
listener.Initialize(tc);
bool result = await listener.CreateSessionAsync(tokenSource.Token);
trace.Info("result: {0}", result);
// Assert.
Assert.True(result);
_runnerServer
.Verify(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is <TaskAgentSession>(y => y != null),
tokenSource.Token), Times.Once());
}
}
public async void DeleteSession()
{
using (TestHostContext tc = CreateTestContext())
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = tc.GetTrace();
// Arrange.
var expectedSession = new TaskAgentSession();
PropertyInfo sessionIdProperty = expectedSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(sessionIdProperty);
sessionIdProperty.SetValue(expectedSession, Guid.NewGuid());
_agentServer
.Setup(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is <TaskAgentSession>(y => y != null),
tokenSource.Token))
.Returns(Task.FromResult(expectedSession));
_capabilitiesManager.Setup(x => x.GetCapabilitiesAsync(_settings, It.IsAny <CancellationToken>())).Returns(Task.FromResult(new Dictionary <string, string>()));
_credMgr.Setup(x => x.LoadCredentials()).Returns(new Common.VssCredentials());
// Act.
MessageListener listener = new MessageListener();
listener.Initialize(tc);
bool result = await listener.CreateSessionAsync(tokenSource.Token);
Assert.True(result);
Assert.Equal(expectedSession, listener.Session);
_agentServer
.Setup(x => x.DeleteAgentSessionAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask);
await listener.DeleteSessionAsync();
//Assert
_agentServer
.Verify(x => x.DeleteAgentSessionAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny <CancellationToken>()), Times.Once());
}
}
public async void CreatesSession()
{
using (TestHostContext tc = CreateTestContext())
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = tc.GetTrace();
// Arrange.
var expectedSession = new TaskAgentSession();
_agentServer
.Setup(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is <TaskAgentSession>(y => y != null),
tokenSource.Token))
.Returns(Task.FromResult(expectedSession));
_capProvider.Setup(x => x.GetCapabilitiesAsync(It.IsAny <string>(), It.IsAny <CancellationToken>())).Returns(Task.FromResult(new Dictionary <string, string>()));
// Act.
MessageListener listener = new MessageListener();
listener.Initialize(tc);
bool result = await listener.CreateSessionAsync(tokenSource.Token);
trace.Info("result: {0}", result);
// Assert.
Assert.True(result);
Assert.Equal(expectedSession, listener.Session);
_agentServer
.Verify(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is <TaskAgentSession>(y => y != null),
tokenSource.Token), Times.Once());
_capProvider.Verify(x => x.GetCapabilitiesAsync(It.IsAny <string>(), It.IsAny <CancellationToken>()));
}
}
public async void CreatesSession()
{
using (TestHostContext tc = CreateTestContext())
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = tc.GetTrace();
// Arrange.
var expectedSession = new TaskAgentSession();
_agentServer
.Setup(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is<TaskAgentSession>(y => y != null),
tokenSource.Token))
.Returns(Task.FromResult(expectedSession));
_capabilitiesManager.Setup(x => x.GetCapabilitiesAsync(_settings, It.IsAny<CancellationToken>())).Returns(Task.FromResult(new Dictionary<string, string>()));
_credMgr.Setup(x => x.LoadCredentials()).Returns(new Common.VssCredentials());
// Act.
MessageListener listener = new MessageListener();
listener.Initialize(tc);
bool result = await listener.CreateSessionAsync(tokenSource.Token);
trace.Info("result: {0}", result);
// Assert.
Assert.True(result);
Assert.Equal(expectedSession, listener.Session);
_agentServer
.Verify(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is<TaskAgentSession>(y => y != null),
tokenSource.Token), Times.Once());
}
}
//process 2 new job messages, and one cancel message
public async void TestRunAsync()
{
using (var hc = new TestHostContext(this))
using (var tokenSource = new CancellationTokenSource())
{
//Arrange
var agent = new Agent.Listener.Agent();
agent.TokenSource = tokenSource;
hc.SetSingleton<IConfigurationManager>(_configurationManager.Object);
hc.SetSingleton<IJobNotification>(_jobNotification.Object);
hc.SetSingleton<IMessageListener>(_messageListener.Object);
hc.SetSingleton<IPromptManager>(_promptManager.Object);
hc.SetSingleton<IAgentServer>(_agentServer.Object);
agent.Initialize(hc);
var settings = new AgentSettings
{
PoolId = 43242
};
var taskAgentSession = new TaskAgentSession();
//we use reflection to achieve this, because "set" is internal
PropertyInfo sessionIdProperty = taskAgentSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(sessionIdProperty);
sessionIdProperty.SetValue(taskAgentSession, Guid.NewGuid());
var message = new TaskAgentMessage()
{
Body = JsonUtility.ToString(CreateJobRequestMessage("job1")),
MessageId = 4234,
MessageType = JobRequestMessage.MessageType
};
var messages = new Queue<TaskAgentMessage>();
messages.Enqueue(message);
var signalWorkerComplete = new SemaphoreSlim(0, 1);
_configurationManager.Setup(x => x.LoadSettings())
.Returns(settings);
_configurationManager.Setup(x => x.IsConfigured())
.Returns(true);
_configurationManager.Setup(x => x.EnsureConfiguredAsync(It.IsAny<CommandSettings>()))
.Returns(Task.CompletedTask);
_messageListener.Setup(x => x.CreateSessionAsync(It.IsAny<CancellationToken>()))
.Returns(Task.FromResult<bool>(true));
_messageListener.Setup(x => x.Session)
.Returns(taskAgentSession);
_messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny<CancellationToken>()))
.Returns(async () =>
{
if (0 == messages.Count)
{
signalWorkerComplete.Release();
await Task.Delay(2000, tokenSource.Token);
}
return messages.Dequeue();
});
_messageListener.Setup(x => x.DeleteSessionAsync())
.Returns(Task.CompletedTask);
_jobDispatcher.Setup(x => x.Run(It.IsAny<JobRequestMessage>()))
.Callback(()=>
{
});
_agentServer.Setup(x => x.DeleteAgentMessageAsync(settings.PoolId, message.MessageId, taskAgentSession.SessionId, It.IsAny<CancellationToken>()))
.Returns((Int32 poolId, Int64 messageId, Guid sessionId, CancellationToken cancellationToken) =>
{
return Task.CompletedTask;
});
_jobNotification.Setup(x => x.StartClient(It.IsAny<String>(), It.IsAny<CancellationToken>()))
.Callback(() =>
{
});
hc.EnqueueInstance<IJobDispatcher>(_jobDispatcher.Object);
//Act
var command = new CommandSettings(hc, new string[0]);
Task agentTask = agent.ExecuteCommand(command);
//Assert
//wait for the agent to run one job
if (!await signalWorkerComplete.WaitAsync(2000))
{
Assert.True(false, $"{nameof(_messageListener.Object.GetNextMessageAsync)} was not invoked.");
}
else
{
//Act
tokenSource.Cancel(); //stop Agent
//Assert
Task[] taskToWait2 = { agentTask, Task.Delay(2000) };
//wait for the Agent to exit
await Task.WhenAny(taskToWait2);
Assert.True(agentTask.IsCompleted, $"{nameof(agent.ExecuteCommand)} timed out.");
Assert.True(!agentTask.IsFaulted, agentTask.Exception?.ToString());
Assert.True(agentTask.IsCanceled);
_jobDispatcher.Verify(x => x.Run(It.IsAny<JobRequestMessage>()), Times.Once(),
$"{nameof(_jobDispatcher.Object.Run)} was not invoked.");
_messageListener.Verify(x => x.GetNextMessageAsync(It.IsAny<CancellationToken>()), Times.AtLeastOnce());
_messageListener.Verify(x => x.CreateSessionAsync(It.IsAny<CancellationToken>()), Times.Once());
_messageListener.Verify(x => x.DeleteSessionAsync(), Times.Once());
_agentServer.Verify(x => x.DeleteAgentMessageAsync(settings.PoolId, message.MessageId, taskAgentSession.SessionId, It.IsAny<CancellationToken>()), Times.AtLeastOnce());
}
}
}
public async void GetNextMessage()
{
using (TestHostContext tc = CreateTestContext())
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = tc.GetTrace();
// Arrange.
var expectedSession = new TaskAgentSession();
PropertyInfo sessionIdProperty = expectedSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(sessionIdProperty);
sessionIdProperty.SetValue(expectedSession, Guid.NewGuid());
_agentServer
.Setup(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is <TaskAgentSession>(y => y != null),
tokenSource.Token))
.Returns(Task.FromResult(expectedSession));
_capabilitiesManager.Setup(x => x.GetCapabilitiesAsync(_settings, It.IsAny <CancellationToken>())).Returns(Task.FromResult(new Dictionary <string, string>()));
_credMgr.Setup(x => x.LoadCredentials()).Returns(new Common.VssCredentials());
// Act.
MessageListener listener = new MessageListener();
listener.Initialize(tc);
bool result = await listener.CreateSessionAsync(tokenSource.Token);
Assert.True(result);
Assert.Equal(expectedSession, listener.Session);
var arMessages = new TaskAgentMessage[]
{
new TaskAgentMessage
{
Body = "somebody1",
MessageId = 4234,
MessageType = JobRequestMessageTypes.AgentJobRequest
},
new TaskAgentMessage
{
Body = "somebody2",
MessageId = 4235,
MessageType = JobCancelMessage.MessageType
},
null, //should be skipped by GetNextMessageAsync implementation
null,
new TaskAgentMessage
{
Body = "somebody3",
MessageId = 4236,
MessageType = JobRequestMessageTypes.AgentJobRequest
}
};
var messages = new Queue <TaskAgentMessage>(arMessages);
_agentServer
.Setup(x => x.GetAgentMessageAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny <long?>(), tokenSource.Token))
.Returns(async(Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken) =>
{
await Task.Yield();
return(messages.Dequeue());
});
TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token);
TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token);
TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token);
Assert.Equal(arMessages[0], message1);
Assert.Equal(arMessages[1], message2);
Assert.Equal(arMessages[4], message3);
//Assert
_agentServer
.Verify(x => x.GetAgentMessageAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny <long?>(), tokenSource.Token), Times.Exactly(arMessages.Length));
}
}
//-----------------------------------------------------------------
// MessageQueue
//-----------------------------------------------------------------
public Task <TaskAgentSession> CreateAgentSessionAsync(Int32 poolId, TaskAgentSession session, CancellationToken cancellationToken)
{
CheckConnection();
return(_taskAgentClient.CreateAgentSessionAsync(poolId, session, cancellationToken: cancellationToken));
}
//-----------------------------------------------------------------
// MessageQueue
//-----------------------------------------------------------------
public Task <TaskAgentSession> CreateAgentSessionAsync(Int32 poolId, TaskAgentSession session, CancellationToken cancellationToken)
{
CheckConnection(RunnerConnectionType.MessageQueue);
return(_messageTaskAgentClient.CreateAgentSessionAsync(poolId, session, cancellationToken: cancellationToken));
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Create connection.
Trace.Info("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = BuildConstants.RunnerPackage.Version,
OSDescription = RuntimeInformation.OSDescription,
};
string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent);
string errorMessage = string.Empty;
bool encounteringError = false;
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Runner Server...");
await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
Trace.Info("VssConnection created");
_term.WriteLine();
_term.WriteSuccessMessage("Connected to GitHub");
_term.WriteLine();
_session = await _runnerServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (ex is VssOAuthTokenRequestException && creds.Federated is VssOAuthCredential vssOAuthCred)
{
// Check whether we get 401 because the runner registration already removed by the service.
// If the runner registration get deleted, we can't exchange oauth token.
Trace.Error("Test oauth app registration.");
var oauthTokenProvider = new VssOAuthTokenProvider(vssOAuthCred, new Uri(serverUrl));
var authError = await oauthTokenProvider.ValidateCredentialAsync(token);
if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
{
_term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure.");
return(false);
}
}
if (!IsSessionCreationExceptionRetriable(ex))
{
_term.WriteError($"Failed to create session. {ex.Message}");
return(false);
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Capabilities.
_term.WriteLine(StringUtil.Loc("ScanToolCapabilities"));
Dictionary <string, string> systemCapabilities = await HostContext.GetService <ICapabilitiesManager>().GetCapabilitiesAsync(_settings, token);
// Create connection.
Trace.Verbose("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
Uri uri = new Uri(serverUrl);
VssConnection conn = ApiUtil.CreateConnection(uri, creds);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = Constants.Agent.Version,
};
string sessionName = $"{Environment.MachineName ?? "AGENT"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent, systemCapabilities);
string errorMessage = string.Empty;
bool encounteringError = false;
_term.WriteLine(StringUtil.Loc("ConnectToServer"));
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Agent Server...");
await _agentServer.ConnectAsync(conn);
_session = await _agentServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine(StringUtil.Loc("QueueConnected", DateTime.UtcNow));
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
_term.WriteError(StringUtil.Loc("SessionCreateFailed", ex.Message));
return(false);
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError(StringUtil.Loc("QueueConError", DateTime.UtcNow, ex.Message, _sessionCreationRetryInterval.TotalSeconds));
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Create connection.
Trace.Info("Loading Credentials");
_useMigratedCredentials = !StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_SPSAUTHURL"));
VssCredentials creds = _credMgr.LoadCredentials(_useMigratedCredentials);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = BuildConstants.RunnerPackage.Version,
OSDescription = RuntimeInformation.OSDescription,
};
string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent);
string errorMessage = string.Empty;
bool encounteringError = false;
var originalCreds = _configStore.GetCredentials();
var migratedCreds = _configStore.GetMigratedCredentials();
if (migratedCreds == null)
{
_useMigratedCredentials = false;
if (originalCreds.Scheme == Constants.Configuration.OAuth)
{
_needToCheckAuthorizationUrlUpdate = true;
}
}
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Runner Server...");
await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
Trace.Info("VssConnection created");
_term.WriteLine();
_term.WriteSuccessMessage("Connected to GitHub");
_term.WriteLine();
_session = await _runnerServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
if (_needToCheckAuthorizationUrlUpdate)
{
// start background task try to get new authorization url
_authorizationUrlMigrationBackgroundTask = GetNewOAuthAuthorizationSetting(token);
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
if (_useMigratedCredentials)
{
// migrated credentials might cause lose permission during permission check,
// we will force to use original credential and try again
_useMigratedCredentials = false;
var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36));
_authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours.
creds = _credMgr.LoadCredentials(false);
Trace.Error("Fallback to original credentials and try again.");
}
else
{
_term.WriteError($"Failed to create session. {ex.Message}");
return(false);
}
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Create connection.
Trace.Info("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = BuildConstants.RunnerPackage.Version,
OSDescription = RuntimeInformation.OSDescription,
};
string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent);
string errorMessage = string.Empty;
bool encounteringError = false;
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Runner Server...");
await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
Trace.Info("VssConnection created");
_term.WriteLine();
_term.WriteSuccessMessage("Connected to GitHub");
_term.WriteLine();
_session = await _runnerServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
_term.WriteError($"Failed to create session. {ex.Message}");
return(false);
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
//process 2 new job messages, and one cancel message
public async void TestRunAsync()
{
using (var hc = new TestHostContext(this))
using (var tokenSource = new CancellationTokenSource())
{
//Arrange
var agent = new Agent.Listener.Agent();
agent.TokenSource = tokenSource;
hc.SetSingleton <IConfigurationManager>(_configurationManager.Object);
hc.SetSingleton <IMessageListener>(_messageListener.Object);
hc.SetSingleton <IPromptManager>(_promptManager.Object);
hc.SetSingleton <IAgentServer>(_agentServer.Object);
agent.Initialize(hc);
var settings = new AgentSettings
{
PoolId = 43242
};
var taskAgentSession = new TaskAgentSession();
//we use reflection to achieve this, because "set" is internal
PropertyInfo sessionIdProperty = taskAgentSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(sessionIdProperty);
sessionIdProperty.SetValue(taskAgentSession, Guid.NewGuid());
var message = new TaskAgentMessage()
{
Body = JsonUtility.ToString(CreateJobRequestMessage("job1")),
MessageId = 4234,
MessageType = JobRequestMessage.MessageType
};
var messages = new Queue <TaskAgentMessage>();
messages.Enqueue(message);
var signalWorkerComplete = new SemaphoreSlim(0, 1);
_configurationManager.Setup(x => x.LoadSettings())
.Returns(settings);
_configurationManager.Setup(x => x.IsConfigured())
.Returns(true);
_configurationManager.Setup(x => x.EnsureConfiguredAsync(It.IsAny <CommandSettings>()))
.Returns(Task.CompletedTask);
_messageListener.Setup(x => x.CreateSessionAsync(It.IsAny <CancellationToken>()))
.Returns(Task.FromResult <bool>(true));
_messageListener.Setup(x => x.Session)
.Returns(taskAgentSession);
_messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny <CancellationToken>()))
.Returns(async() =>
{
if (0 == messages.Count)
{
signalWorkerComplete.Release();
await Task.Delay(2000, tokenSource.Token);
}
return(messages.Dequeue());
});
_messageListener.Setup(x => x.DeleteSessionAsync())
.Returns(Task.CompletedTask);
_jobDispatcher.Setup(x => x.Run(It.IsAny <JobRequestMessage>()))
.Callback(() =>
{
});
_agentServer.Setup(x => x.DeleteAgentMessageAsync(settings.PoolId, message.MessageId, taskAgentSession.SessionId, It.IsAny <CancellationToken>()))
.Returns((Int32 poolId, Int64 messageId, Guid sessionId, CancellationToken cancellationToken) =>
{
return(Task.CompletedTask);
});
hc.EnqueueInstance <IJobDispatcher>(_jobDispatcher.Object);
//Act
var command = new CommandSettings(hc, new string[0]);
Task agentTask = agent.ExecuteCommand(command);
//Assert
//wait for the agent to run one job
if (!await signalWorkerComplete.WaitAsync(2000))
{
Assert.True(false, $"{nameof(_messageListener.Object.GetNextMessageAsync)} was not invoked.");
}
else
{
//Act
tokenSource.Cancel(); //stop Agent
//Assert
Task[] taskToWait2 = { agentTask, Task.Delay(2000) };
//wait for the Agent to exit
await Task.WhenAny(taskToWait2);
Assert.True(agentTask.IsCompleted, $"{nameof(agent.ExecuteCommand)} timed out.");
Assert.True(!agentTask.IsFaulted, agentTask.Exception?.ToString());
Assert.True(agentTask.IsCanceled);
_jobDispatcher.Verify(x => x.Run(It.IsAny <JobRequestMessage>()), Times.Once(),
$"{nameof(_jobDispatcher.Object.Run)} was not invoked.");
_messageListener.Verify(x => x.GetNextMessageAsync(It.IsAny <CancellationToken>()), Times.AtLeastOnce());
_messageListener.Verify(x => x.CreateSessionAsync(It.IsAny <CancellationToken>()), Times.Once());
_messageListener.Verify(x => x.DeleteSessionAsync(), Times.Once());
_agentServer.Verify(x => x.DeleteAgentMessageAsync(settings.PoolId, message.MessageId, taskAgentSession.SessionId, It.IsAny <CancellationToken>()), Times.AtLeastOnce());
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
const int MaxAttempts = 10;
int attempt = 0;
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
int agentPoolId = _settings.PoolId;
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Load Credentials
Trace.Verbose("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
Uri uri = new Uri(serverUrl);
VssConnection conn = ApiUtil.CreateConnection(uri, creds);
string sessionName = $"{Environment.MachineName ?? string.Empty}_{Guid.NewGuid().ToString()}";
var capProvider = HostContext.GetService <ICapabilitiesProvider>();
Dictionary <string, string> agentSystemCapabilities = await capProvider.GetCapabilitiesAsync(_settings.AgentName, token);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = Constants.Agent.Version,
Enabled = true
};
var taskAgentSession = new TaskAgentSession(sessionName, agent, agentSystemCapabilities);
var agentSvr = HostContext.GetService <IAgentServer>();
while (++attempt <= MaxAttempts)
{
Trace.Info("Create session attempt {0} of {1}.", attempt, MaxAttempts);
try
{
Trace.Info("Connecting to the Agent Server...");
await agentSvr.ConnectAsync(conn);
Session = await agentSvr.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
return(true);
}
catch (OperationCanceledException)
{
Trace.Info("Cancelled");
throw;
}
catch (Exception ex)
{
Trace.Error("Failed to create session.");
if (ex is TaskAgentNotFoundException)
{
Trace.Error("The agent no longer exists on the server. Stopping the agent.");
Trace.Error(ex);
return(false);
}
else if (ex is TaskAgentSessionConflictException)
{
Trace.Error("The session for this agent already exists.");
}
else
{
Trace.Error(ex);
}
if (attempt >= MaxAttempts)
{
Trace.Error("Retries exhausted. Terminating the agent.");
return(false);
}
TimeSpan interval = TimeSpan.FromSeconds(30);
Trace.Info("Sleeping for {0} seconds before retrying.", interval.TotalSeconds);
await HostContext.Delay(interval, token);
}
}
return(false);
}
public async Task<Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService<IConfigurationManager>();
_settings = configManager.LoadSettings();
int agentPoolId = _settings.PoolId;
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Capabilities.
// TODO: LOC
_term.WriteLine("Scanning for tool capabilities.");
Dictionary<string, string> systemCapabilities = await HostContext.GetService<ICapabilitiesManager>().GetCapabilitiesAsync(_settings, token);
// Create connection.
Trace.Verbose("Loading Credentials");
var credMgr = HostContext.GetService<ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
Uri uri = new Uri(serverUrl);
VssConnection conn = ApiUtil.CreateConnection(uri, creds);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = Constants.Agent.Version,
};
string sessionName = $"{Environment.MachineName ?? "AGENT"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent, systemCapabilities);
var agentSvr = HostContext.GetService<IAgentServer>();
string errorMessage = string.Empty;
bool encounteringError = false;
// TODO: LOC
_term.WriteLine("Connecting to the server.");
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Agent Server...");
await agentSvr.ConnectAsync(conn);
Session = await agentSvr.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine(StringUtil.Loc("QueueConnected", DateTime.UtcNow));
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
return true;
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
_term.WriteError(StringUtil.Loc("SessionCreateFailed", ex.Message));
return false;
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError(StringUtil.Loc("QueueConError", DateTime.UtcNow, ex.Message, _sessionCreationRetryInterval.TotalSeconds));
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
int attempt = 0;
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
int agentPoolId = _settings.PoolId;
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Capabilities.
// TODO: LOC
_term.WriteLine("Scanning for tool capabilities.");
Dictionary <string, string> systemCapabilities = await HostContext.GetService <ICapabilitiesManager>().GetCapabilitiesAsync(_settings, token);
// Create connection.
Trace.Verbose("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
Uri uri = new Uri(serverUrl);
VssConnection conn = ApiUtil.CreateConnection(uri, creds);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = Constants.Agent.Version,
Enabled = true
};
string sessionName = $"{Environment.MachineName ?? "AGENT"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent, systemCapabilities);
var agentSvr = HostContext.GetService <IAgentServer>();
string errorMessage = string.Empty;
bool firstAttempt = true; //tells us if this is the first time we try to connect
// TODO: LOC
_term.WriteLine("Connecting to the server.");
while (true)
{
attempt++;
Trace.Info($"Create session attempt {attempt}.");
try
{
Trace.Info("Connecting to the Agent Server...");
await agentSvr.ConnectAsync(conn);
Session = await agentSvr.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
if (!firstAttempt)
{
_term.WriteLine(StringUtil.Loc("QueueConnected", DateTime.UtcNow));
}
return(true);
}
catch (OperationCanceledException ex)
{
if (token.IsCancellationRequested) //Distinguish timeout from user cancellation
{
Trace.Info("Cancelled");
throw;
}
errorMessage = ex.Message;
}
catch (Exception ex)
{
Trace.Error("Failed to create session.");
if (ex is TaskAgentNotFoundException)
{
Trace.Error("The agent no longer exists on the server. Stopping the agent.");
_term.WriteError(StringUtil.Loc("MissingAgent"));
}
if (ex is TaskAgentSessionConflictException)
{
Trace.Error("The session for this agent already exists.");
_term.WriteError(StringUtil.Loc("SessionExist"));
}
Trace.Error(ex);
if (IsFatalException(ex))
{
_term.WriteError(StringUtil.Loc("SessionCreateFailed"));
return(false);
}
errorMessage = ex.Message;
}
TimeSpan interval = TimeSpan.FromSeconds(30);
if (firstAttempt) //print the message only on the first error
{
_term.WriteError(StringUtil.Loc("QueueConError", DateTime.UtcNow, errorMessage, interval.TotalSeconds));
firstAttempt = false;
}
Trace.Info("Sleeping for {0} seconds before retrying.", interval.TotalSeconds);
await HostContext.Delay(interval, token);
}
}
public async void DeleteSession()
{
using (TestHostContext tc = CreateTestContext())
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = tc.GetTrace();
// Arrange.
var expectedSession = new TaskAgentSession();
PropertyInfo sessionIdProperty = expectedSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(sessionIdProperty);
sessionIdProperty.SetValue(expectedSession, Guid.NewGuid());
_agentServer
.Setup(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is<TaskAgentSession>(y => y != null),
tokenSource.Token))
.Returns(Task.FromResult(expectedSession));
_capabilitiesManager.Setup(x => x.GetCapabilitiesAsync(_settings, It.IsAny<CancellationToken>())).Returns(Task.FromResult(new Dictionary<string, string>()));
_credMgr.Setup(x => x.LoadCredentials()).Returns(new Common.VssCredentials());
// Act.
MessageListener listener = new MessageListener();
listener.Initialize(tc);
bool result = await listener.CreateSessionAsync(tokenSource.Token);
Assert.True(result);
Assert.Equal(expectedSession, listener.Session);
_agentServer
.Setup(x => x.DeleteAgentSessionAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny<CancellationToken>()))
.Returns(Task.CompletedTask);
await listener.DeleteSessionAsync();
//Assert
_agentServer
.Verify(x => x.DeleteAgentSessionAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny<CancellationToken>()), Times.Once());
}
}
public async void GetNextMessage()
{
using (TestHostContext tc = CreateTestContext())
using (var tokenSource = new CancellationTokenSource())
{
Tracing trace = tc.GetTrace();
// Arrange.
var expectedSession = new TaskAgentSession();
PropertyInfo sessionIdProperty = expectedSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(sessionIdProperty);
sessionIdProperty.SetValue(expectedSession, Guid.NewGuid());
_agentServer
.Setup(x => x.CreateAgentSessionAsync(
_settings.PoolId,
It.Is<TaskAgentSession>(y => y != null),
tokenSource.Token))
.Returns(Task.FromResult(expectedSession));
_capabilitiesManager.Setup(x => x.GetCapabilitiesAsync(_settings, It.IsAny<CancellationToken>())).Returns(Task.FromResult(new Dictionary<string, string>()));
_credMgr.Setup(x => x.LoadCredentials()).Returns(new Common.VssCredentials());
// Act.
MessageListener listener = new MessageListener();
listener.Initialize(tc);
bool result = await listener.CreateSessionAsync(tokenSource.Token);
Assert.True(result);
Assert.Equal(expectedSession, listener.Session);
var arMessages = new TaskAgentMessage[]
{
new TaskAgentMessage
{
Body = "somebody1",
MessageId = 4234,
MessageType = JobRequestMessage.MessageType
},
new TaskAgentMessage
{
Body = "somebody2",
MessageId = 4235,
MessageType = JobCancelMessage.MessageType
},
null, //should be skipped by GetNextMessageAsync implementation
null,
new TaskAgentMessage
{
Body = "somebody3",
MessageId = 4236,
MessageType = JobRequestMessage.MessageType
}
};
var messages = new Queue<TaskAgentMessage>(arMessages);
_agentServer
.Setup(x => x.GetAgentMessageAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny<long?>(), tokenSource.Token))
.Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken) =>
{
await Task.Yield();
return messages.Dequeue();
});
TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token);
TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token);
TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token);
Assert.Equal(arMessages[0], message1);
Assert.Equal(arMessages[1], message2);
Assert.Equal(arMessages[4], message3);
//Assert
_agentServer
.Verify(x => x.GetAgentMessageAsync(
_settings.PoolId, expectedSession.SessionId, It.IsAny<long?>(), tokenSource.Token), Times.Exactly(arMessages.Length));
}
}
//-----------------------------------------------------------------
// MessageQueue
//-----------------------------------------------------------------
public Task<TaskAgentSession> CreateAgentSessionAsync(Int32 poolId, TaskAgentSession session, CancellationToken cancellationToken)
{
CheckConnection();
return _taskAgentClient.CreateAgentSessionAsync(poolId, session, null, cancellationToken);
}