public async void RenewJobRequestNewAgentNameUpdatesSettings()
{
//Arrange
using (var hc = new TestHostContext(this))
{
var count = 0;
var oldName = "OldName";
var newName = "NewName";
var oldSettings = new RunnerSettings {
AgentName = oldName
};
var reservedAgent = new TaskAgentReference {
Name = newName
};
var trace = hc.GetTrace(nameof(DispatcherRenewJobRequestStopOnJobTokenExpiredExceptions));
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
var request = new Mock <TaskAgentJobRequest>();
request.Object.ReservedAgent = reservedAgent;
PropertyInfo lockUntilProperty = request.Object.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(lockUntilProperty);
lockUntilProperty.SetValue(request.Object, DateTime.UtcNow.AddMinutes(5));
hc.SetSingleton <IRunnerServer>(_runnerServer.Object);
hc.SetSingleton <IConfigurationStore>(_configurationStore.Object);
_configurationStore.Setup(x => x.GetSettings()).Returns(oldSettings);
_runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny <int>(), It.IsAny <long>(), It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.Returns(() =>
{
count++;
if (count < 5)
{
return(Task.FromResult <TaskAgentJobRequest>(request.Object));
}
else if (count == 5 || count == 6 || count == 7)
{
throw new TimeoutException("");
}
else
{
cancellationTokenSource.Cancel();
return(Task.FromResult <TaskAgentJobRequest>(request.Object));
}
});
var jobDispatcher = new JobDispatcher();
jobDispatcher.Initialize(hc);
// Act
await jobDispatcher.RenewJobRequestAsync(0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
// Assert
_configurationStore.Verify(x => x.SaveSettings(It.Is <RunnerSettings>(settings => settings.AgentName == newName)), Times.Once);
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Create connection.
Trace.Info("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = BuildConstants.RunnerPackage.Version,
OSDescription = RuntimeInformation.OSDescription,
};
string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent);
string errorMessage = string.Empty;
bool encounteringError = false;
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Runner Server...");
await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
Trace.Info("VssConnection created");
_term.WriteLine();
_term.WriteSuccessMessage("Connected to GitHub");
_term.WriteLine();
_session = await _runnerServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (ex is VssOAuthTokenRequestException && creds.Federated is VssOAuthCredential vssOAuthCred)
{
// Check whether we get 401 because the runner registration already removed by the service.
// If the runner registration get deleted, we can't exchange oauth token.
Trace.Error("Test oauth app registration.");
var oauthTokenProvider = new VssOAuthTokenProvider(vssOAuthCred, new Uri(serverUrl));
var authError = await oauthTokenProvider.ValidateCredentialAsync(token);
if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
{
_term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure.");
return(false);
}
}
if (!IsSessionCreationExceptionRetriable(ex))
{
_term.WriteError($"Failed to create session. {ex.Message}");
return(false);
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Capabilities.
_term.WriteLine(StringUtil.Loc("ScanToolCapabilities"));
Dictionary <string, string> systemCapabilities = await HostContext.GetService <ICapabilitiesManager>().GetCapabilitiesAsync(_settings, token);
// Create connection.
Trace.Verbose("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
Uri uri = new Uri(serverUrl);
VssConnection conn = ApiUtil.CreateConnection(uri, creds);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = Constants.Agent.Version,
};
string sessionName = $"{Environment.MachineName ?? "AGENT"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent, systemCapabilities);
string errorMessage = string.Empty;
bool encounteringError = false;
_term.WriteLine(StringUtil.Loc("ConnectToServer"));
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Agent Server...");
await _agentServer.ConnectAsync(conn);
_session = await _agentServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine(StringUtil.Loc("QueueConnected", DateTime.UtcNow));
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
_term.WriteError(StringUtil.Loc("SessionCreateFailed", ex.Message));
return(false);
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError(StringUtil.Loc("QueueConError", DateTime.UtcNow, ex.Message, _sessionCreationRetryInterval.TotalSeconds));
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Create connection.
Trace.Info("Loading Credentials");
_useMigratedCredentials = !StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_SPSAUTHURL"));
VssCredentials creds = _credMgr.LoadCredentials(_useMigratedCredentials);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = BuildConstants.RunnerPackage.Version,
OSDescription = RuntimeInformation.OSDescription,
};
string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent);
string errorMessage = string.Empty;
bool encounteringError = false;
var originalCreds = _configStore.GetCredentials();
var migratedCreds = _configStore.GetMigratedCredentials();
if (migratedCreds == null)
{
_useMigratedCredentials = false;
if (originalCreds.Scheme == Constants.Configuration.OAuth)
{
_needToCheckAuthorizationUrlUpdate = true;
}
}
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Runner Server...");
await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
Trace.Info("VssConnection created");
_term.WriteLine();
_term.WriteSuccessMessage("Connected to GitHub");
_term.WriteLine();
_session = await _runnerServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
if (_needToCheckAuthorizationUrlUpdate)
{
// start background task try to get new authorization url
_authorizationUrlMigrationBackgroundTask = GetNewOAuthAuthorizationSetting(token);
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
if (_useMigratedCredentials)
{
// migrated credentials might cause lose permission during permission check,
// we will force to use original credential and try again
_useMigratedCredentials = false;
var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36));
_authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours.
creds = _credMgr.LoadCredentials(false);
Trace.Error("Fallback to original credentials and try again.");
}
else
{
_term.WriteError($"Failed to create session. {ex.Message}");
return(false);
}
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Create connection.
Trace.Info("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = BuildConstants.RunnerPackage.Version,
OSDescription = RuntimeInformation.OSDescription,
};
string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent);
string errorMessage = string.Empty;
bool encounteringError = false;
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Runner Server...");
await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
Trace.Info("VssConnection created");
_term.WriteLine();
_term.WriteSuccessMessage("Connected to GitHub");
_term.WriteLine();
_session = await _runnerServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
_term.WriteError($"Failed to create session. {ex.Message}");
return(false);
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
const int MaxAttempts = 10;
int attempt = 0;
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
int agentPoolId = _settings.PoolId;
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Load Credentials
Trace.Verbose("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
Uri uri = new Uri(serverUrl);
VssConnection conn = ApiUtil.CreateConnection(uri, creds);
string sessionName = $"{Environment.MachineName ?? string.Empty}_{Guid.NewGuid().ToString()}";
var capProvider = HostContext.GetService <ICapabilitiesProvider>();
Dictionary <string, string> agentSystemCapabilities = await capProvider.GetCapabilitiesAsync(_settings.AgentName, token);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = Constants.Agent.Version,
Enabled = true
};
var taskAgentSession = new TaskAgentSession(sessionName, agent, agentSystemCapabilities);
var agentSvr = HostContext.GetService <IAgentServer>();
while (++attempt <= MaxAttempts)
{
Trace.Info("Create session attempt {0} of {1}.", attempt, MaxAttempts);
try
{
Trace.Info("Connecting to the Agent Server...");
await agentSvr.ConnectAsync(conn);
Session = await agentSvr.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
return(true);
}
catch (OperationCanceledException)
{
Trace.Info("Cancelled");
throw;
}
catch (Exception ex)
{
Trace.Error("Failed to create session.");
if (ex is TaskAgentNotFoundException)
{
Trace.Error("The agent no longer exists on the server. Stopping the agent.");
Trace.Error(ex);
return(false);
}
else if (ex is TaskAgentSessionConflictException)
{
Trace.Error("The session for this agent already exists.");
}
else
{
Trace.Error(ex);
}
if (attempt >= MaxAttempts)
{
Trace.Error("Retries exhausted. Terminating the agent.");
return(false);
}
TimeSpan interval = TimeSpan.FromSeconds(30);
Trace.Info("Sleeping for {0} seconds before retrying.", interval.TotalSeconds);
await HostContext.Delay(interval, token);
}
}
return(false);
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
int attempt = 0;
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
int agentPoolId = _settings.PoolId;
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Capabilities.
// TODO: LOC
_term.WriteLine("Scanning for tool capabilities.");
Dictionary <string, string> systemCapabilities = await HostContext.GetService <ICapabilitiesManager>().GetCapabilitiesAsync(_settings, token);
// Create connection.
Trace.Verbose("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
Uri uri = new Uri(serverUrl);
VssConnection conn = ApiUtil.CreateConnection(uri, creds);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = Constants.Agent.Version,
Enabled = true
};
string sessionName = $"{Environment.MachineName ?? "AGENT"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent, systemCapabilities);
var agentSvr = HostContext.GetService <IAgentServer>();
string errorMessage = string.Empty;
bool firstAttempt = true; //tells us if this is the first time we try to connect
// TODO: LOC
_term.WriteLine("Connecting to the server.");
while (true)
{
attempt++;
Trace.Info($"Create session attempt {attempt}.");
try
{
Trace.Info("Connecting to the Agent Server...");
await agentSvr.ConnectAsync(conn);
Session = await agentSvr.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
if (!firstAttempt)
{
_term.WriteLine(StringUtil.Loc("QueueConnected", DateTime.UtcNow));
}
return(true);
}
catch (OperationCanceledException ex)
{
if (token.IsCancellationRequested) //Distinguish timeout from user cancellation
{
Trace.Info("Cancelled");
throw;
}
errorMessage = ex.Message;
}
catch (Exception ex)
{
Trace.Error("Failed to create session.");
if (ex is TaskAgentNotFoundException)
{
Trace.Error("The agent no longer exists on the server. Stopping the agent.");
_term.WriteError(StringUtil.Loc("MissingAgent"));
}
if (ex is TaskAgentSessionConflictException)
{
Trace.Error("The session for this agent already exists.");
_term.WriteError(StringUtil.Loc("SessionExist"));
}
Trace.Error(ex);
if (IsFatalException(ex))
{
_term.WriteError(StringUtil.Loc("SessionCreateFailed"));
return(false);
}
errorMessage = ex.Message;
}
TimeSpan interval = TimeSpan.FromSeconds(30);
if (firstAttempt) //print the message only on the first error
{
_term.WriteError(StringUtil.Loc("QueueConError", DateTime.UtcNow, errorMessage, interval.TotalSeconds));
firstAttempt = false;
}
Trace.Info("Sleeping for {0} seconds before retrying.", interval.TotalSeconds);
await HostContext.Delay(interval, token);
}
}
public async Task<Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService<IConfigurationManager>();
_settings = configManager.LoadSettings();
int agentPoolId = _settings.PoolId;
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Capabilities.
// TODO: LOC
_term.WriteLine("Scanning for tool capabilities.");
Dictionary<string, string> systemCapabilities = await HostContext.GetService<ICapabilitiesManager>().GetCapabilitiesAsync(_settings, token);
// Create connection.
Trace.Verbose("Loading Credentials");
var credMgr = HostContext.GetService<ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
Uri uri = new Uri(serverUrl);
VssConnection conn = ApiUtil.CreateConnection(uri, creds);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = Constants.Agent.Version,
};
string sessionName = $"{Environment.MachineName ?? "AGENT"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent, systemCapabilities);
var agentSvr = HostContext.GetService<IAgentServer>();
string errorMessage = string.Empty;
bool encounteringError = false;
// TODO: LOC
_term.WriteLine("Connecting to the server.");
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Agent Server...");
await agentSvr.ConnectAsync(conn);
Session = await agentSvr.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine(StringUtil.Loc("QueueConnected", DateTime.UtcNow));
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
return true;
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
_term.WriteError(StringUtil.Loc("SessionCreateFailed", ex.Message));
return false;
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError(StringUtil.Loc("QueueConError", DateTime.UtcNow, ex.Message, _sessionCreationRetryInterval.TotalSeconds));
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
