public void PublishEvent(IExecutionContext context, CustomerIntelligenceEvent ciEvent)
{
ICustomerIntelligenceServer ciService;
VssConnection vssConnection;
try
{
ciService = context.GetHostContext().GetService <ICustomerIntelligenceServer>();
vssConnection = WorkerUtilities.GetVssConnection(context);
ciService.Initialize(vssConnection);
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, WorkerUtilities.GetVssConnection(context).Uri.ToString(), context.Warning);
return;
}
catch (Exception ex)
{
context.Warning(StringUtil.Loc("TelemetryCommandFailed", ex.Message));
return;
}
var commandContext = context.GetHostContext().CreateService <IAsyncCommandContext>();
commandContext.InitializeCommandContext(context, StringUtil.Loc("Telemetry"));
commandContext.Task = PublishEventsAsync(context, ciService, ciEvent);
}
public async Task <Stream> GetFileTask(ContainerItem ticketedItem, CancellationToken cancellationToken)
{
ArgUtil.NotNull(ticketedItem, nameof(ticketedItem));
this._executionContext.Debug(StringUtil.Format("Get file container client for file {0}", ticketedItem.Path));
VssConnection vssConnection = await GetVssConnection();
FileContainerHttpClient fileContainer = null;
try
{
fileContainer = vssConnection.GetClient <FileContainerHttpClient>();
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, vssConnection.Uri.ToString(), this._executionContext.Error);
throw;
}
this._executionContext.Debug(StringUtil.Format("Start fetch file stream from filecontainer service for file {0}", ticketedItem.Path));
Stream stream = await fileContainer.DownloadFileAsync(
ticketedItem.ContainerId,
ticketedItem.Path,
cancellationToken,
scopeIdentifier : ticketedItem.ScopeIdentifier);
this._executionContext.Debug(StringUtil.Format("Finished fetch file stream from filecontainer service for file {0}", ticketedItem.Path));
return(stream);
}
public async Task ConnectAsync(VssConnection jobConnection)
{
ArgUtil.NotNull(jobConnection, nameof(jobConnection));
_connection = jobConnection;
int attemptCount = 5;
while (!_connection.HasAuthenticated && attemptCount-- > 0)
{
try
{
await _connection.ConnectAsync();
break;
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, _connection.Uri.ToString(), Trace.Error);
}
catch (Exception ex) when(attemptCount > 0)
{
Trace.Info($"Catch exception during connect. {attemptCount} attemp left.");
Trace.Error(ex);
}
await Task.Delay(100);
}
_releaseHttpClient = _connection.GetClient <ReleaseHttpClient>();
}
/// <inheritdoc />
public async Task <bool> InitializeAsync(IAgentLogPluginContext context)
{
try
{
_logger = _logger ?? new TraceLogger(context);
_telemetry = _telemetry ?? new TelemetryDataCollector(new ClientFactory(context.VssConnection), _logger);
await PopulatePipelineConfig(context);
if (DisablePlugin(context))
{
_telemetry.AddOrUpdate(TelemetryConstants.PluginDisabled, true);
await _telemetry.PublishCumulativeTelemetryAsync();
return(false); // disable the plugin
}
_testFilePublisher = _testFilePublisher ??
new TestFilePublisher(context.VssConnection, PipelineConfig, new TestFileTraceListener(context), _logger, _telemetry);
await _testFilePublisher.InitializeAsync();
_telemetry.AddOrUpdate(TelemetryConstants.PluginInitialized, true);
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, context.VssConnection.Uri.ToString(), _logger.Warning);
if (_telemetry != null)
{
_telemetry.AddOrUpdate(TelemetryConstants.PluginDisabled, true);
_telemetry.AddOrUpdate(TelemetryConstants.InitializeFailed, ex);
await _telemetry.PublishCumulativeTelemetryAsync();
}
return(false);
}
catch (Exception ex)
{
context.Trace(ex.ToString());
_logger?.Warning($"Unable to initialize {FriendlyName}.");
if (_telemetry != null)
{
_telemetry.AddOrUpdate(TelemetryConstants.PluginDisabled, true);
_telemetry.AddOrUpdate(TelemetryConstants.InitializeFailed, ex);
await _telemetry.PublishCumulativeTelemetryAsync();
}
return(false);
}
return(true);
}
private Uri GetTenantAuthorityUrl(IHostContext context, string serverUrl)
{
Tracing trace = context.GetTrace(nameof(AadDeviceCodeAccessToken));
using (var handler = context.CreateHttpClientHandler())
using (var client = new HttpClient(handler))
{
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Add("X-TFS-FedAuthRedirect", "Suppress");
client.DefaultRequestHeaders.UserAgent.Clear();
client.DefaultRequestHeaders.UserAgent.AddRange(VssClientHttpRequestSettings.Default.UserAgent);
using (var requestMessage = new HttpRequestMessage(HttpMethod.Head, $"{serverUrl.Trim('/')}/_apis/connectiondata"))
{
HttpResponseMessage response;
try
{
response = client.SendAsync(requestMessage).GetAwaiter().GetResult();
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, serverUrl, trace.Error);
throw;
}
// Get the tenant from the Login URL, MSA backed accounts will not return `Bearer` www-authenticate header.
var bearerResult = response.Headers.WwwAuthenticate.Where(p => p.Scheme.Equals("Bearer", StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (bearerResult != null && bearerResult.Parameter.StartsWith("authorization_uri=", StringComparison.OrdinalIgnoreCase))
{
var authorizationUri = bearerResult.Parameter.Substring("authorization_uri=".Length);
if (Uri.TryCreate(authorizationUri, UriKind.Absolute, out Uri aadTenantUrl))
{
return(aadTenantUrl);
}
}
return(null);
}
}
}
private async Task DownloadArtifactsAndCommitsAsync(IExecutionContext executionContext, object data)
{
Trace.Entering();
try
{
await DownloadArtifacts(executionContext, ReleaseArtifacts, ArtifactsWorkingFolder);
await DownloadCommits(executionContext, TeamProjectId, ReleaseArtifacts);
}
catch (SocketException ex)
{
LogDownloadFailureTelemetry(executionContext, ex);
ExceptionsUtil.HandleSocketException(ex, WorkerUtilities.GetVssConnection(executionContext).Uri.ToString(), Trace.Error);
throw;
}
catch (Exception ex)
{
LogDownloadFailureTelemetry(executionContext, ex);
throw;
}
}
private async Task <string> GetAzureSubscriptionIdAsync()
{
// We will use the Azure Instance Metadata Service in order to fetch metadata ( in this case Subscription Id used to provision the VM) if the VM is an Azure VM
// More on Instance Metadata Service can be found here: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service
string azureSubscriptionId = string.Empty;
const string imdsUri = "http://169.254.169.254/metadata/instance/compute/subscriptionId?api-version=2017-08-01&format=text";
using (var handler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(handler))
{
httpClient.DefaultRequestHeaders.Add("Metadata", "True");
httpClient.Timeout = TimeSpan.FromSeconds(5);
try
{
azureSubscriptionId = await httpClient.GetStringAsync(imdsUri);
if (!Guid.TryParse(azureSubscriptionId, out Guid result))
{
azureSubscriptionId = string.Empty;
}
}
catch (SocketException ex)
{
azureSubscriptionId = string.Empty;
ExceptionsUtil.HandleSocketException(ex, imdsUri, Trace.Info);
}
catch (Exception ex)
{
// An exception will be thrown if the Agent Machine is a non-Azure VM.
azureSubscriptionId = string.Empty;
Trace.Info($"GetAzureSubscriptionId ex: {ex.Message}");
}
}
return(azureSubscriptionId);
}
public async Task ConnectAsync(VssConnection jobConnection)
{
ArgUtil.NotNull(jobConnection, nameof(jobConnection));
_connection = jobConnection;
try
{
await _connection.ConnectAsync();
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, _connection.Uri.ToString(), Trace.Error);
throw;
}
catch (Exception ex)
{
Trace.Info($"Unable to connect to {_connection.Uri}.");
Trace.Error(ex);
throw;
}
_locationClient = _connection.GetClient <LocationHttpClient>();
_hasConnection = true;
}
public static int Main(string[] args)
{
if (PlatformUtil.UseLegacyHttpHandler)
{
AppContext.SetSwitch("System.Net.Http.UseSocketsHttpHandler", false);
}
Console.CancelKeyPress += Console_CancelKeyPress;
// Set encoding to UTF8, process invoker will use UTF8 write to STDIN
Console.InputEncoding = Encoding.UTF8;
Console.OutputEncoding = Encoding.UTF8;
try
{
ArgUtil.NotNull(args, nameof(args));
ArgUtil.Equal(2, args.Length, nameof(args.Length));
string pluginType = args[0];
if (string.Equals("task", pluginType, StringComparison.OrdinalIgnoreCase))
{
string assemblyQualifiedName = args[1];
ArgUtil.NotNullOrEmpty(assemblyQualifiedName, nameof(assemblyQualifiedName));
string serializedContext = Console.ReadLine();
ArgUtil.NotNullOrEmpty(serializedContext, nameof(serializedContext));
AgentTaskPluginExecutionContext executionContext = StringUtil.ConvertFromJson <AgentTaskPluginExecutionContext>(serializedContext);
ArgUtil.NotNull(executionContext, nameof(executionContext));
VariableValue culture;
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
if (executionContext.Variables.TryGetValue("system.culture", out culture) &&
!string.IsNullOrEmpty(culture?.Value))
{
CultureInfo.DefaultThreadCurrentCulture = new CultureInfo(culture.Value);
CultureInfo.DefaultThreadCurrentUICulture = new CultureInfo(culture.Value);
}
AssemblyLoadContext.Default.Resolving += ResolveAssembly;
try
{
Type type = Type.GetType(assemblyQualifiedName, throwOnError: true);
var taskPlugin = Activator.CreateInstance(type) as IAgentTaskPlugin;
ArgUtil.NotNull(taskPlugin, nameof(taskPlugin));
taskPlugin.RunAsync(executionContext, tokenSource.Token).GetAwaiter().GetResult();
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, executionContext.VssConnection.Uri.ToString(), executionContext.Error);
}
catch (AggregateException ex)
{
ExceptionsUtil.HandleAggregateException((AggregateException)ex, executionContext.Error);
}
catch (Exception ex)
{
executionContext.Error(ex.Message);
executionContext.Debug(ex.StackTrace);
}
finally
{
AssemblyLoadContext.Default.Resolving -= ResolveAssembly;
}
return(0);
}
else if (string.Equals("command", pluginType, StringComparison.OrdinalIgnoreCase))
{
string assemblyQualifiedName = args[1];
ArgUtil.NotNullOrEmpty(assemblyQualifiedName, nameof(assemblyQualifiedName));
string serializedContext = Console.ReadLine();
ArgUtil.NotNullOrEmpty(serializedContext, nameof(serializedContext));
AgentCommandPluginExecutionContext executionContext = StringUtil.ConvertFromJson <AgentCommandPluginExecutionContext>(serializedContext);
ArgUtil.NotNull(executionContext, nameof(executionContext));
AssemblyLoadContext.Default.Resolving += ResolveAssembly;
try
{
Type type = Type.GetType(assemblyQualifiedName, throwOnError: true);
var commandPlugin = Activator.CreateInstance(type) as IAgentCommandPlugin;
ArgUtil.NotNull(commandPlugin, nameof(commandPlugin));
commandPlugin.ProcessCommandAsync(executionContext, tokenSource.Token).GetAwaiter().GetResult();
}
catch (Exception ex)
{
// any exception throw from plugin will fail the command.
executionContext.Error(ex.ToString());
}
finally
{
AssemblyLoadContext.Default.Resolving -= ResolveAssembly;
}
return(0);
}
else if (string.Equals("log", pluginType, StringComparison.OrdinalIgnoreCase))
{
// read commandline arg to get the instance id
var instanceId = args[1];
ArgUtil.NotNullOrEmpty(instanceId, nameof(instanceId));
// read STDIN, the first line will be the HostContext for the log plugin host
string serializedContext = Console.ReadLine();
ArgUtil.NotNullOrEmpty(serializedContext, nameof(serializedContext));
AgentLogPluginHostContext hostContext = StringUtil.ConvertFromJson <AgentLogPluginHostContext>(serializedContext);
ArgUtil.NotNull(hostContext, nameof(hostContext));
// create plugin object base on plugin assembly names from the HostContext
List <IAgentLogPlugin> logPlugins = new List <IAgentLogPlugin>();
AssemblyLoadContext.Default.Resolving += ResolveAssembly;
try
{
foreach (var pluginAssembly in hostContext.PluginAssemblies)
{
try
{
Type type = Type.GetType(pluginAssembly, throwOnError: true);
var logPlugin = Activator.CreateInstance(type) as IAgentLogPlugin;
ArgUtil.NotNull(logPlugin, nameof(logPlugin));
logPlugins.Add(logPlugin);
}
catch (Exception ex)
{
// any exception throw from plugin will get trace and ignore, error from plugins will not fail the job.
Console.WriteLine($"Unable to load plugin '{pluginAssembly}': {ex}");
}
}
}
finally
{
AssemblyLoadContext.Default.Resolving -= ResolveAssembly;
}
// start the log plugin host
var logPluginHost = new AgentLogPluginHost(hostContext, logPlugins);
Task hostTask = logPluginHost.Run();
while (true)
{
var consoleInput = Console.ReadLine();
if (string.Equals(consoleInput, $"##vso[logplugin.finish]{instanceId}", StringComparison.OrdinalIgnoreCase))
{
// singal all plugins, the job has finished.
// plugin need to start their finalize process.
logPluginHost.Finish();
break;
}
else
{
// the format is TimelineRecordId(GUID):Output(String)
logPluginHost.EnqueueOutput(consoleInput);
}
}
// wait for the host to finish.
hostTask.GetAwaiter().GetResult();
return(0);
}
else
{
throw new ArgumentOutOfRangeException(pluginType);
}
}
catch (Exception ex)
{
// infrastructure failure.
Console.Error.WriteLine(ex.ToString());
return(1);
}
finally
{
Console.CancelKeyPress -= Console_CancelKeyPress;
}
}
public async Task ConfigureAsync(CommandSettings command)
{
ArgUtil.NotNull(command, nameof(command));
if (PlatformUtil.RunningOnWindows)
{
CheckAgentRootDirectorySecure();
}
Trace.Info(nameof(ConfigureAsync));
if (IsConfigured())
{
throw new InvalidOperationException(StringUtil.Loc("AlreadyConfiguredError"));
}
// Populate proxy setting from commandline args
var vstsProxy = HostContext.GetService <IVstsAgentWebProxy>();
bool saveProxySetting = false;
string proxyUrl = command.GetProxyUrl();
if (!string.IsNullOrEmpty(proxyUrl))
{
if (!Uri.IsWellFormedUriString(proxyUrl, UriKind.Absolute))
{
throw new ArgumentOutOfRangeException(nameof(proxyUrl));
}
Trace.Info("Reset proxy base on commandline args.");
string proxyUserName = command.GetProxyUserName();
string proxyPassword = command.GetProxyPassword();
(vstsProxy as VstsAgentWebProxy).SetupProxy(proxyUrl, proxyUserName, proxyPassword);
saveProxySetting = true;
}
// Populate cert setting from commandline args
var agentCertManager = HostContext.GetService <IAgentCertificateManager>();
bool saveCertSetting = false;
bool skipCertValidation = command.GetSkipCertificateValidation();
string caCert = command.GetCACertificate();
string clientCert = command.GetClientCertificate();
string clientCertKey = command.GetClientCertificatePrivateKey();
string clientCertArchive = command.GetClientCertificateArchrive();
string clientCertPassword = command.GetClientCertificatePassword();
// We require all Certificate files are under agent root.
// So we can set ACL correctly when configure as service
if (!string.IsNullOrEmpty(caCert))
{
caCert = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), caCert);
ArgUtil.File(caCert, nameof(caCert));
}
if (!string.IsNullOrEmpty(clientCert) &&
!string.IsNullOrEmpty(clientCertKey) &&
!string.IsNullOrEmpty(clientCertArchive))
{
// Ensure all client cert pieces are there.
clientCert = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), clientCert);
clientCertKey = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), clientCertKey);
clientCertArchive = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), clientCertArchive);
ArgUtil.File(clientCert, nameof(clientCert));
ArgUtil.File(clientCertKey, nameof(clientCertKey));
ArgUtil.File(clientCertArchive, nameof(clientCertArchive));
}
else if (!string.IsNullOrEmpty(clientCert) ||
!string.IsNullOrEmpty(clientCertKey) ||
!string.IsNullOrEmpty(clientCertArchive))
{
// Print out which args are missing.
ArgUtil.NotNullOrEmpty(Constants.Agent.CommandLine.Args.SslClientCert, Constants.Agent.CommandLine.Args.SslClientCert);
ArgUtil.NotNullOrEmpty(Constants.Agent.CommandLine.Args.SslClientCertKey, Constants.Agent.CommandLine.Args.SslClientCertKey);
ArgUtil.NotNullOrEmpty(Constants.Agent.CommandLine.Args.SslClientCertArchive, Constants.Agent.CommandLine.Args.SslClientCertArchive);
}
if (skipCertValidation || !string.IsNullOrEmpty(caCert) || !string.IsNullOrEmpty(clientCert))
{
Trace.Info("Reset agent cert setting base on commandline args.");
(agentCertManager as AgentCertificateManager).SetupCertificate(skipCertValidation, caCert, clientCert, clientCertKey, clientCertArchive, clientCertPassword);
saveCertSetting = true;
}
AgentSettings agentSettings = new AgentSettings();
// TEE EULA
agentSettings.AcceptTeeEula = false;
switch (PlatformUtil.HostOS)
{
case PlatformUtil.OS.OSX:
case PlatformUtil.OS.Linux:
// Write the section header.
WriteSection(StringUtil.Loc("EulasSectionHeader"));
// Verify the EULA exists on disk in the expected location.
string eulaFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), "license.html");
ArgUtil.File(eulaFile, nameof(eulaFile));
// Write elaborate verbiage about the TEE EULA.
_term.WriteLine(StringUtil.Loc("TeeEula", eulaFile));
_term.WriteLine();
// Prompt to acccept the TEE EULA.
agentSettings.AcceptTeeEula = command.GetAcceptTeeEula();
break;
case PlatformUtil.OS.Windows:
// Warn and continue if .NET 4.6 is not installed.
if (!NetFrameworkUtil.Test(new Version(4, 6), Trace))
{
WriteSection(StringUtil.Loc("PrerequisitesSectionHeader")); // Section header.
_term.WriteLine(StringUtil.Loc("MinimumNetFrameworkTfvc")); // Warning.
}
break;
default:
throw new NotSupportedException();
}
// Create the configuration provider as per agent type.
string agentType;
if (command.GetDeploymentOrMachineGroup())
{
agentType = Constants.Agent.AgentConfigurationProvider.DeploymentAgentConfiguration;
}
else if (command.GetDeploymentPool())
{
agentType = Constants.Agent.AgentConfigurationProvider.SharedDeploymentAgentConfiguration;
}
else if (command.GetEnvironmentVMResource())
{
agentType = Constants.Agent.AgentConfigurationProvider.EnvironmentVMResourceConfiguration;
}
else
{
agentType = Constants.Agent.AgentConfigurationProvider.BuildReleasesAgentConfiguration;
}
var extensionManager = HostContext.GetService <IExtensionManager>();
IConfigurationProvider agentProvider =
(extensionManager.GetExtensions <IConfigurationProvider>())
.FirstOrDefault(x => x.ConfigurationProviderType == agentType);
ArgUtil.NotNull(agentProvider, agentType);
bool isHostedServer = false;
// Loop getting url and creds until you can connect
ICredentialProvider credProvider = null;
VssCredentials creds = null;
WriteSection(StringUtil.Loc("ConnectSectionHeader"));
while (true)
{
// Get the URL
agentProvider.GetServerUrl(agentSettings, command);
// Get the credentials
credProvider = GetCredentialProvider(command, agentSettings.ServerUrl);
creds = credProvider.GetVssCredentials(HostContext);
Trace.Info("cred retrieved");
try
{
// Determine the service deployment type based on connection data. (Hosted/OnPremises)
await _serverUtil.DetermineDeploymentType(agentSettings.ServerUrl, creds, _locationServer);
if (!_serverUtil.TryGetDeploymentType(out isHostedServer))
{
Trace.Warning(@"Deployment type determination has been failed;
assume it is OnPremises and the deployment type determination was not implemented for this server version.");
}
// Get the collection name for deployment group
agentProvider.GetCollectionName(agentSettings, command, isHostedServer);
// Validate can connect.
await agentProvider.TestConnectionAsync(agentSettings, creds, isHostedServer);
Trace.Info("Test Connection complete.");
break;
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, agentSettings.ServerUrl, _term.WriteError);
}
catch (Exception e) when(!command.Unattended())
{
_term.WriteError(e);
_term.WriteError(StringUtil.Loc("FailedToConnect"));
}
}
// We want to use the native CSP of the platform for storage, so we use the RSACSP directly
RSAParameters publicKey;
var keyManager = HostContext.GetService <IRSAKeyManager>();
using (var rsa = keyManager.CreateKey())
{
publicKey = rsa.ExportParameters(false);
}
// Loop getting agent name and pool name
WriteSection(StringUtil.Loc("RegisterAgentSectionHeader"));
while (true)
{
try
{
await agentProvider.GetPoolIdAndName(agentSettings, command);
break;
}
catch (Exception e) when(!command.Unattended())
{
_term.WriteError(e);
_term.WriteError(agentProvider.GetFailedToFindPoolErrorString());
}
}
TaskAgent agent;
while (true)
{
agentSettings.AgentName = command.GetAgentName();
// Get the system capabilities.
// TODO: Hook up to ctrl+c cancellation token.
_term.WriteLine(StringUtil.Loc("ScanToolCapabilities"));
Dictionary <string, string> systemCapabilities = await HostContext.GetService <ICapabilitiesManager>().GetCapabilitiesAsync(agentSettings, CancellationToken.None);
_term.WriteLine(StringUtil.Loc("ConnectToServer"));
agent = await agentProvider.GetAgentAsync(agentSettings);
if (agent != null)
{
if (command.GetReplace())
{
// Update existing agent with new PublicKey, agent version and SystemCapabilities.
agent = UpdateExistingAgent(agent, publicKey, systemCapabilities);
try
{
agent = await agentProvider.UpdateAgentAsync(agentSettings, agent, command);
_term.WriteLine(StringUtil.Loc("AgentReplaced"));
break;
}
catch (Exception e) when(!command.Unattended())
{
_term.WriteError(e);
_term.WriteError(StringUtil.Loc("FailedToReplaceAgent"));
}
}
else if (command.Unattended())
{
// if not replace and it is unattended config.
agentProvider.ThrowTaskAgentExistException(agentSettings);
}
}
else
{
// Create a new agent.
agent = CreateNewAgent(agentSettings.AgentName, publicKey, systemCapabilities);
try
{
agent = await agentProvider.AddAgentAsync(agentSettings, agent, command);
_term.WriteLine(StringUtil.Loc("AgentAddedSuccessfully"));
break;
}
catch (Exception e) when(!command.Unattended())
{
_term.WriteError(e);
_term.WriteError(StringUtil.Loc("AddAgentFailed"));
}
}
}
// Add Agent Id to settings
agentSettings.AgentId = agent.Id;
// respect the serverUrl resolve by server.
// in case of agent configured using collection url instead of account url.
string agentServerUrl;
if (agent.Properties.TryGetValidatedValue <string>("ServerUrl", out agentServerUrl) &&
!string.IsNullOrEmpty(agentServerUrl))
{
Trace.Info($"Agent server url resolve by server: '{agentServerUrl}'.");
// we need make sure the Schema/Host/Port component of the url remain the same.
UriBuilder inputServerUrl = new UriBuilder(agentSettings.ServerUrl);
UriBuilder serverReturnedServerUrl = new UriBuilder(agentServerUrl);
if (Uri.Compare(inputServerUrl.Uri, serverReturnedServerUrl.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
{
inputServerUrl.Path = serverReturnedServerUrl.Path;
Trace.Info($"Replace server returned url's scheme://host:port component with user input server url's scheme://host:port: '{inputServerUrl.Uri.AbsoluteUri}'.");
agentSettings.ServerUrl = inputServerUrl.Uri.AbsoluteUri;
}
else
{
agentSettings.ServerUrl = agentServerUrl;
}
}
// See if the server supports our OAuth key exchange for credentials
if (agent.Authorization != null &&
agent.Authorization.ClientId != Guid.Empty &&
agent.Authorization.AuthorizationUrl != null)
{
// We use authorizationUrl as the oauth endpoint url by default.
// For TFS, we need make sure the Schema/Host/Port component of the oauth endpoint url also match configuration url. (Incase of customer's agent configure URL and TFS server public URL are different)
// Which means, we will keep use the original authorizationUrl in the VssOAuthJwtBearerClientCredential (authorizationUrl is the audience),
// But might have different Url in VssOAuthCredential (connection url)
// We can't do this for VSTS, since its SPS/TFS urls are different.
UriBuilder configServerUrl = new UriBuilder(agentSettings.ServerUrl);
UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl);
if (!isHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
{
oauthEndpointUrlBuilder.Scheme = configServerUrl.Scheme;
oauthEndpointUrlBuilder.Host = configServerUrl.Host;
oauthEndpointUrlBuilder.Port = configServerUrl.Port;
Trace.Info($"Set oauth endpoint url's scheme://host:port component to match agent configure url's scheme://host:port: '{oauthEndpointUrlBuilder.Uri.AbsoluteUri}'.");
}
var credentialData = new CredentialData
{
Scheme = Constants.Configuration.OAuth,
Data =
{
{ "clientId", agent.Authorization.ClientId.ToString("D") },
{ "authorizationUrl", agent.Authorization.AuthorizationUrl.AbsoluteUri },
{ "oauthEndpointUrl", oauthEndpointUrlBuilder.Uri.AbsoluteUri },
},
};
// Save the negotiated OAuth credential data
_store.SaveCredential(credentialData);
}
else
{
switch (PlatformUtil.HostOS)
{
case PlatformUtil.OS.OSX:
case PlatformUtil.OS.Linux:
// Save the provided admin cred for compat with previous agent.
_store.SaveCredential(credProvider.CredentialData);
break;
case PlatformUtil.OS.Windows:
// Not supported against TFS 2015.
_term.WriteError(StringUtil.Loc("Tfs2015NotSupported"));
return;
default:
throw new NotSupportedException();
}
}
// Testing agent connection, detect any protential connection issue, like local clock skew that cause OAuth token expired.
_term.WriteLine(StringUtil.Loc("TestAgentConnection"));
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials credential = credMgr.LoadCredentials();
var agentSvr = HostContext.GetService <IAgentServer>();
try
{
await agentSvr.ConnectAsync(new Uri(agentSettings.ServerUrl), credential);
}
catch (VssOAuthTokenRequestException ex) when(ex.Message.Contains("Current server time is"))
{
// there are two exception messages server send that indicate clock skew.
// 1. The bearer token expired on {jwt.ValidTo}. Current server time is {DateTime.UtcNow}.
// 2. The bearer token is not valid until {jwt.ValidFrom}. Current server time is {DateTime.UtcNow}.
Trace.Error("Catch exception during test agent connection.");
Trace.Error(ex);
throw new InvalidOperationException(StringUtil.Loc("LocalClockSkewed"));
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, agentSettings.ServerUrl, Trace.Error);
throw;
}
// We will Combine() what's stored with root. Defaults to string a relative path
agentSettings.WorkFolder = command.GetWork();
// notificationPipeName for Hosted agent provisioner.
agentSettings.NotificationPipeName = command.GetNotificationPipeName();
agentSettings.MonitorSocketAddress = command.GetMonitorSocketAddress();
agentSettings.NotificationSocketAddress = command.GetNotificationSocketAddress();
agentSettings.DisableLogUploads = command.GetDisableLogUploads();
agentSettings.AlwaysExtractTask = command.GetAlwaysExtractTask();
_store.SaveSettings(agentSettings);
if (saveProxySetting)
{
Trace.Info("Save proxy setting to disk.");
(vstsProxy as VstsAgentWebProxy).SaveProxySetting();
}
if (saveCertSetting)
{
Trace.Info("Save agent cert setting to disk.");
(agentCertManager as AgentCertificateManager).SaveCertificateSetting();
}
_term.WriteLine(StringUtil.Loc("SavedSettings", DateTime.UtcNow));
bool saveRuntimeOptions = false;
var runtimeOptions = new AgentRuntimeOptions();
if (PlatformUtil.RunningOnWindows && command.GetGitUseSChannel())
{
saveRuntimeOptions = true;
runtimeOptions.GitUseSecureChannel = true;
}
if (saveRuntimeOptions)
{
Trace.Info("Save agent runtime options to disk.");
_store.SaveAgentRuntimeOptions(runtimeOptions);
}
if (PlatformUtil.RunningOnWindows)
{
// config windows service
bool runAsService = command.GetRunAsService();
if (runAsService)
{
Trace.Info("Configuring to run the agent as service");
var serviceControlManager = HostContext.GetService <IWindowsServiceControlManager>();
agentSettings.EnableServiceSidTypeUnrestricted = command.GetEnableServiceSidTypeUnrestricted();
serviceControlManager.ConfigureService(agentSettings, command);
}
// config auto logon
else if (command.GetRunAsAutoLogon())
{
Trace.Info("Agent is going to run as process setting up the 'AutoLogon' capability for the agent.");
var autoLogonConfigManager = HostContext.GetService <IAutoLogonManager>();
await autoLogonConfigManager.ConfigureAsync(command);
//Important: The machine may restart if the autologon user is not same as the current user
//if you are adding code after this, keep that in mind
}
}
else if (PlatformUtil.RunningOnLinux)
{
// generate service config script for Linux
var serviceControlManager = HostContext.GetService <ILinuxServiceControlManager>();
serviceControlManager.GenerateScripts(agentSettings);
}
else if (PlatformUtil.RunningOnMacOS)
{
// generate service config script for macOS
var serviceControlManager = HostContext.GetService <IMacOSServiceControlManager>();
serviceControlManager.GenerateScripts(agentSettings);
}
}
public async Task UnconfigureAsync(CommandSettings command)
{
ArgUtil.NotNull(command, nameof(command));
string currentAction = string.Empty;
try
{
//stop, uninstall service and remove service config file
if (_store.IsServiceConfigured())
{
currentAction = StringUtil.Loc("UninstallingService");
_term.WriteLine(currentAction);
if (PlatformUtil.RunningOnWindows)
{
var serviceControlManager = HostContext.GetService <IWindowsServiceControlManager>();
serviceControlManager.UnconfigureService();
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
}
else if (PlatformUtil.RunningOnLinux)
{
// unconfig systemd service first
throw new InvalidOperationException(StringUtil.Loc("UnconfigureServiceDService"));
}
else if (PlatformUtil.RunningOnMacOS)
{
// unconfig macOS service first
throw new InvalidOperationException(StringUtil.Loc("UnconfigureOSXService"));
}
}
else
{
if (PlatformUtil.RunningOnWindows)
{
//running as process, unconfigure autologon if it was configured
if (_store.IsAutoLogonConfigured())
{
currentAction = StringUtil.Loc("UnconfigAutologon");
_term.WriteLine(currentAction);
var autoLogonConfigManager = HostContext.GetService <IAutoLogonManager>();
autoLogonConfigManager.Unconfigure();
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
}
else
{
Trace.Info("AutoLogon was not configured on the agent.");
}
}
}
//delete agent from the server
currentAction = StringUtil.Loc("UnregisteringAgent");
_term.WriteLine(currentAction);
bool isConfigured = _store.IsConfigured();
bool hasCredentials = _store.HasCredentials();
if (isConfigured && hasCredentials)
{
AgentSettings settings = _store.GetSettings();
var credentialManager = HostContext.GetService <ICredentialManager>();
// Get the credentials
var credProvider = GetCredentialProvider(command, settings.ServerUrl);
VssCredentials creds = credProvider.GetVssCredentials(HostContext);
Trace.Info("cred retrieved");
bool isEnvironmentVMResource = false;
bool isDeploymentGroup = (settings.MachineGroupId > 0) || (settings.DeploymentGroupId > 0);
if (!isDeploymentGroup)
{
isEnvironmentVMResource = settings.EnvironmentId > 0;
}
Trace.Info("Agent configured for deploymentGroup : {0}", isDeploymentGroup.ToString());
string agentType = isDeploymentGroup
? Constants.Agent.AgentConfigurationProvider.DeploymentAgentConfiguration
: isEnvironmentVMResource
? Constants.Agent.AgentConfigurationProvider.EnvironmentVMResourceConfiguration
: Constants.Agent.AgentConfigurationProvider.BuildReleasesAgentConfiguration;
var extensionManager = HostContext.GetService <IExtensionManager>();
IConfigurationProvider agentProvider = (extensionManager.GetExtensions <IConfigurationProvider>()).FirstOrDefault(x => x.ConfigurationProviderType == agentType);
ArgUtil.NotNull(agentProvider, agentType);
// Determine the service deployment type based on connection data. (Hosted/OnPremises)
bool isHostedServer;
await _serverUtil.DetermineDeploymentType(settings.ServerUrl, creds, _locationServer);
if (!_serverUtil.TryGetDeploymentType(out isHostedServer))
{
Trace.Warning(@"Deployment type determination has been failed;
assume it is OnPremises and the deployment type determination was not implemented for this server version.");
}
await agentProvider.TestConnectionAsync(settings, creds, isHostedServer);
TaskAgent agent = await agentProvider.GetAgentAsync(settings);
if (agent == null)
{
_term.WriteLine(StringUtil.Loc("Skipping") + currentAction);
}
else
{
await agentProvider.DeleteAgentAsync(settings);
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
}
}
else
{
_term.WriteLine(StringUtil.Loc("MissingConfig"));
}
//delete credential config files
currentAction = StringUtil.Loc("DeletingCredentials");
_term.WriteLine(currentAction);
if (hasCredentials)
{
_store.DeleteCredential();
var keyManager = HostContext.GetService <IRSAKeyManager>();
keyManager.DeleteKey();
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
}
else
{
_term.WriteLine(StringUtil.Loc("Skipping") + currentAction);
}
//delete settings config file
currentAction = StringUtil.Loc("DeletingSettings");
_term.WriteLine(currentAction);
if (isConfigured)
{
// delete proxy setting
(HostContext.GetService <IVstsAgentWebProxy>() as VstsAgentWebProxy).DeleteProxySetting();
// delete agent cert setting
(HostContext.GetService <IAgentCertificateManager>() as AgentCertificateManager).DeleteCertificateSetting();
// delete agent runtime option
_store.DeleteAgentRuntimeOptions();
_store.DeleteSettings();
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
}
else
{
_term.WriteLine(StringUtil.Loc("Skipping") + currentAction);
}
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, _store.GetSettings().ServerUrl, _term.WriteLine);
throw;
}
catch (Exception)
{
_term.WriteLine(StringUtil.Loc("Failed") + currentAction);
throw;
}
}
/// <summary>
/// _work
/// \_update
/// \bin
/// \externals
/// \run.sh
/// \run.cmd
/// \package.zip //temp download .zip/.tar.gz
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
private async Task DownloadLatestAgent(CancellationToken token)
{
string latestAgentDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.UpdateDirectory);
IOUtil.DeleteDirectory(latestAgentDirectory, token);
Directory.CreateDirectory(latestAgentDirectory);
int agentSuffix = 1;
string archiveFile = null;
bool downloadSucceeded = false;
bool validationSucceeded = false;
try
{
// Download the agent, using multiple attempts in order to be resilient against any networking/CDN issues
for (int attempt = 1; attempt <= Constants.AgentDownloadRetryMaxAttempts && !validationSucceeded; attempt++)
{
// Generate an available package name, and do our best effort to clean up stale local zip files
while (true)
{
if (_targetPackage.Platform.StartsWith("win"))
{
archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.zip");
}
else
{
archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.tar.gz");
}
// The package name is generated, check if there is already a file with the same name and path
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip '{0}'", archiveFile);
try
{
// Such a file already exists, so try deleting it
IOUtil.DeleteFile(archiveFile);
// The file was successfully deleted, so we can use the generated package name
break;
}
catch (Exception ex)
{
// Couldn't delete the file for whatever reason, so generate another package name
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
agentSuffix++;
}
}
else
{
// There is no a file with the same name and path, so we can use the generated package name
break;
}
}
// Allow a 15-minute package download timeout, which is good enough to update the agent from a 1 Mbit/s ADSL connection.
var timeoutSeconds = AgentKnobs.AgentDownloadTimeout.GetValue(_knobContext).AsInt();
Trace.Info($"Attempt {attempt}: save latest agent into {archiveFile}.");
using (var downloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var downloadCts = CancellationTokenSource.CreateLinkedTokenSource(downloadTimeout.Token, token))
{
try
{
Trace.Info($"Download agent: begin download");
//open zip stream in async mode
using (var handler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(handler))
using (var fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
using (var result = await httpClient.GetStreamAsync(_targetPackage.DownloadUrl))
{
//81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
await result.CopyToAsync(fs, 81920, downloadCts.Token);
await fs.FlushAsync(downloadCts.Token);
}
Trace.Info($"Download agent: finished download");
downloadSucceeded = true;
validationSucceeded = HashValidation(archiveFile);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info($"Agent download has been canceled.");
throw;
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, _targetPackage.DownloadUrl, Trace.Warning);
}
catch (Exception ex)
{
if (downloadCts.Token.IsCancellationRequested)
{
Trace.Warning($"Agent download has timed out after {timeoutSeconds} seconds");
}
Trace.Warning($"Failed to get package '{archiveFile}' from '{_targetPackage.DownloadUrl}'. Exception {ex}");
}
}
}
if (!downloadSucceeded)
{
throw new TaskCanceledException($"Agent package '{archiveFile}' failed after {Constants.AgentDownloadRetryMaxAttempts} download attempts.");
}
if (!validationSucceeded)
{
throw new TaskCanceledException(@"Agent package checksum validation failed.
There are possible reasons why this happened:
1) The agent package was compromised.
2) The agent package was not fully downloaded or was corrupted during the download process.
You can skip checksum validation for the agent package by setting the environment variable DISABLE_HASH_VALIDATION=true");
}
// If we got this far, we know that we've successfully downloadeded the agent package
if (archiveFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
ZipFile.ExtractToDirectory(archiveFile, latestAgentDirectory);
}
else if (archiveFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase))
{
string tar = WhichUtil.Which("tar", trace: Trace);
if (string.IsNullOrEmpty(tar))
{
throw new NotSupportedException($"tar -xzf");
}
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(latestAgentDirectory, tar, $"-xzf \"{archiveFile}\"", null, token);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
}
else
{
throw new NotSupportedException($"{archiveFile}");
}
Trace.Info($"Finished getting latest agent package at: {latestAgentDirectory}.");
}
finally
{
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip: {0}", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the .zip file
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
}
}
if (!String.IsNullOrEmpty(AgentKnobs.DisableAuthenticodeValidation.GetValue(HostContext).AsString()))
{
Trace.Warning("Authenticode validation skipped for downloaded agent package since it is disabled currently by agent settings.");
}
else
{
if (PlatformUtil.RunningOnWindows)
{
var isValid = this.VerifyAgentAuthenticode(latestAgentDirectory);
if (!isValid)
{
throw new Exception("Authenticode validation of agent assemblies failed.");
}
else
{
Trace.Info("Authenticode validation of agent assemblies passed successfully.");
}
}
else
{
Trace.Info("Authenticode validation skipped since it's not supported on non-Windows platforms at the moment.");
}
}
// copy latest agent into agent root folder
// copy bin from _work/_update -> bin.version under root
string binVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.BinDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(binVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory)} to {binVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory), binVersionDir, token);
// copy externals from _work/_update -> externals.version under root
string externalsVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.ExternalsDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(externalsVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory)} to {externalsVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory), externalsVersionDir, token);
// copy and replace all .sh/.cmd files
Trace.Info($"Copy any remaining .sh/.cmd files into agent root.");
foreach (FileInfo file in new DirectoryInfo(latestAgentDirectory).GetFiles() ?? new FileInfo[0])
{
// Copy and replace the file.
file.CopyTo(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name), true);
}
// for windows service back compat with old windows agent, we need make sure the servicehost.exe is still the old name
// if the current bin folder has VsoAgentService.exe, then the new agent bin folder needs VsoAgentService.exe as well
if (PlatformUtil.RunningOnWindows)
{
if (File.Exists(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "VsoAgentService.exe")))
{
Trace.Info($"Make a copy of AgentService.exe, name it VsoAgentService.exe");
File.Copy(Path.Combine(binVersionDir, "AgentService.exe"), Path.Combine(binVersionDir, "VsoAgentService.exe"), true);
File.Copy(Path.Combine(binVersionDir, "AgentService.exe.config"), Path.Combine(binVersionDir, "VsoAgentService.exe.config"), true);
Trace.Info($"Make a copy of Agent.Listener.exe, name it VsoAgent.exe");
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.exe"), Path.Combine(binVersionDir, "VsoAgent.exe"), true);
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.dll"), Path.Combine(binVersionDir, "VsoAgent.dll"), true);
// in case of we remove all pdb file from agent package.
if (File.Exists(Path.Combine(binVersionDir, "AgentService.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "AgentService.pdb"), Path.Combine(binVersionDir, "VsoAgentService.pdb"), true);
}
if (File.Exists(Path.Combine(binVersionDir, "Agent.Listener.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.pdb"), Path.Combine(binVersionDir, "VsoAgent.pdb"), true);
}
}
}
}
private async Task <UploadResult> BlobUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token)
{
// return files that fail to upload and total artifact size
var uploadResult = new UploadResult();
// nothing needs to upload
if (files.Count == 0)
{
return(uploadResult);
}
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
try
{
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
int maxParallelism = context.GetHostContext().GetService <IConfigurationStore>().GetSettings().MaxDedupParallelism;
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, maxParallelism, token);
// Upload to blobstore
var results = await BlobStoreUtils.UploadBatchToBlobstore(verbose, files, (level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(BlobUploadAsync), context), (str) => context.Output(str), dedupClient, clientTelemetry, token, enableReporting : true);
// Associate with TFS
context.Output(StringUtil.Loc("AssociateFiles"));
var queue = new ConcurrentQueue <BlobFileInfo>();
foreach (var file in results.fileDedupIds)
{
queue.Enqueue(file);
}
// Start associate monitor
var uploadFinished = new TaskCompletionSource <int>();
var associateMonitor = AssociateReportingAsync(context, files.Count(), uploadFinished, token);
// Start parallel associate tasks.
var parallelAssociateTasks = new List <Task <UploadResult> >();
for (int uploader = 0; uploader < concurrentUploads; uploader++)
{
parallelAssociateTasks.Add(AssociateAsync(context, queue, token));
}
// Wait for parallel associate tasks to finish.
await Task.WhenAll(parallelAssociateTasks);
foreach (var associateTask in parallelAssociateTasks)
{
// record all failed files.
uploadResult.AddUploadResult(await associateTask);
}
// Stop monitor task
uploadFinished.SetResult(0);
await associateMonitor;
// report telemetry
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.PlanId), out var planId))
{
planId = Guid.Empty;
}
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.JobId), out var jobId))
{
jobId = Guid.Empty;
}
await clientTelemetry.CommitTelemetryUpload(planId, jobId);
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, this._connection.Uri.ToString(), context.Warn);
throw;
}
catch
{
var blobStoreHost = dedupClient.Client.BaseAddress.Host;
var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform();
var warningMessage = StringUtil.Loc("BlobStoreUploadWarning", blobStoreHost, allowListLink);
context.Warn(warningMessage);
throw;
}
return(uploadResult);
}
public IArtifactDetails GetArtifactDetails(
IExecutionContext context,
AgentArtifactDefinition agentArtifactDefinition)
{
ArgUtil.NotNull(context, nameof(context));
ArgUtil.NotNull(agentArtifactDefinition, nameof(agentArtifactDefinition));
var artifactDetails =
JsonConvert.DeserializeObject <Dictionary <string, string> >(agentArtifactDefinition.Details);
string connectionName;
string repositoryName = string.Empty;
string branch = string.Empty;
if (artifactDetails.TryGetValue(ArtifactDefinitionConstants.ConnectionName, out connectionName) &&
artifactDetails.TryGetValue(ArtifactDefinitionConstants.RepositoryId, out repositoryName) &&
artifactDetails.TryGetValue(ArtifactDefinitionConstants.BranchId, out branch))
{
string checkoutNestedSubmodules;
string checkoutSubmodules;
string gitLfsSupport;
string fetchDepth;
artifactDetails.TryGetValue("checkoutNestedSubmodules", out checkoutNestedSubmodules);
artifactDetails.TryGetValue("checkoutSubmodules", out checkoutSubmodules);
artifactDetails.TryGetValue("gitLfsSupport", out gitLfsSupport);
artifactDetails.TryGetValue("fetchDepth", out fetchDepth);
ServiceEndpoint gitHubEndpoint = context.Endpoints.FirstOrDefault((e => string.Equals(e.Name, connectionName, StringComparison.OrdinalIgnoreCase)));
if (gitHubEndpoint == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMGitHubEndpointNotFound", agentArtifactDefinition.Name));
}
string accessToken = gitHubEndpoint.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
GitHubRepository repository = null;
try
{
repository = HostContext.GetService <IGitHubHttpClient>().GetUserRepo(accessToken, repositoryName);
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, "https://api.github.com", Trace.Info);
throw;
}
Trace.Info($"Found github repository url {repository.Clone_url}");
return(new GitHubArtifactDetails
{
RelativePath = Path.DirectorySeparatorChar.ToString(),
ConnectionName = connectionName,
CloneUrl = new Uri(repository.Clone_url),
Branch = branch,
CheckoutSubmodules = checkoutSubmodules,
CheckoutNestedSubmodules = checkoutNestedSubmodules,
GitLfsSupport = gitLfsSupport,
FetchDepth = fetchDepth
});
}
else
{
throw new InvalidOperationException(StringUtil.Loc("RMArtifactDetailsIncomplete"));
}
}
private async Task DownloadFileContainerAsync(IEnumerable <FileContainerItem> items, ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
// Only initialize these clients if we know we need to download from Blobstore
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
{
try
{
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
false, (str) => this.tracer.Info(str), this.connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken);
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, connection.Uri.ToString(), context.Warning);
}
catch
{
var blobStoreHost = dedupClient.Client.BaseAddress.Host;
var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform();
var warningMessage = StringUtil.Loc("BlobStoreDownloadWarning", blobStoreHost, allowListLink);
// Fall back to streaming through TFS if we cannot reach blobstore
downloadFromBlob = false;
tracer.Warn(warningMessage);
}
}
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null && downloadFromBlob)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// Send results to CustomerIntelligence
if (clientTelemetry != null)
{
var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString());
var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString());
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload,
properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId));
}
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
private async Task PublishCodeCoverageAsync(
IExecutionContext executionContext,
IAsyncCommandContext commandContext,
ICodeCoveragePublisher codeCoveragePublisher,
IEnumerable <CodeCoverageStatistics> coverageData,
string project,
Guid projectId,
long containerId,
CancellationToken cancellationToken)
{
//step 2: publish code coverage summary to TFS
if (coverageData != null && coverageData.Count() > 0)
{
commandContext.Output(StringUtil.Loc("PublishingCodeCoverage"));
foreach (var coverage in coverageData)
{
commandContext.Output(StringUtil.Format(" {0}- {1} of {2} covered.", coverage.Label, coverage.Covered, coverage.Total));
}
await codeCoveragePublisher.PublishCodeCoverageSummaryAsync(commandContext, coverageData, project, cancellationToken);
}
// step 3: publish code coverage files as build artifacts
string additionalCodeCoverageFilePath = null;
string destinationSummaryFile = null;
var newReportDirectory = _reportDirectory;
try
{
var filesToPublish = new List <Tuple <string, string> >();
if (!Directory.Exists(newReportDirectory))
{
if (!string.IsNullOrWhiteSpace(newReportDirectory))
{
// user gave a invalid report directory. Write warning and continue.
executionContext.Warning(StringUtil.Loc("DirectoryNotFound", newReportDirectory));
}
newReportDirectory = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.ReportDirectory);
Directory.CreateDirectory(newReportDirectory);
}
var summaryFileName = Path.GetFileName(_summaryFileLocation);
destinationSummaryFile = Path.Combine(newReportDirectory, CodeCoverageConstants.SummaryFileDirectory + _buildId, summaryFileName);
Directory.CreateDirectory(Path.GetDirectoryName(destinationSummaryFile));
File.Copy(_summaryFileLocation, destinationSummaryFile, true);
commandContext.Output(StringUtil.Loc("ModifyingCoberturaIndexFile"));
ModifyCoberturaIndexDotHTML(newReportDirectory, executionContext);
filesToPublish.Add(new Tuple <string, string>(newReportDirectory, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.ReportDirectory)));
if (_additionalCodeCoverageFiles != null && _additionalCodeCoverageFiles.Count != 0)
{
additionalCodeCoverageFilePath = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory);
CodeCoverageUtilities.CopyFilesFromFileListWithDirStructure(_additionalCodeCoverageFiles, ref additionalCodeCoverageFilePath);
filesToPublish.Add(new Tuple <string, string>(additionalCodeCoverageFilePath, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory)));
}
commandContext.Output(StringUtil.Loc("PublishingCodeCoverageFiles"));
ChangeHtmExtensionToHtmlIfRequired(newReportDirectory, executionContext);
await codeCoveragePublisher.PublishCodeCoverageFilesAsync(commandContext, projectId, executionContext.Variables.System_JobId, containerId, filesToPublish, File.Exists(Path.Combine(newReportDirectory, CodeCoverageConstants.DefaultIndexFile)), cancellationToken);
}
catch (SocketException ex)
{
#pragma warning disable CA2000 // Dispose objects before losing scope
ExceptionsUtil.HandleSocketException(ex, WorkerUtilities.GetVssConnection(executionContext).Uri.ToString(), executionContext.Warning);
#pragma warning restore CA2000 // Dispose objects before losing scope
}
catch (Exception ex)
{
executionContext.Warning(StringUtil.Loc("ErrorOccurredWhilePublishingCCFiles", ex.Message));
}
finally
{
// clean temporary files.
if (!string.IsNullOrEmpty(additionalCodeCoverageFilePath))
{
if (Directory.Exists(additionalCodeCoverageFilePath))
{
Directory.Delete(path: additionalCodeCoverageFilePath, recursive: true);
}
}
if (!string.IsNullOrEmpty(destinationSummaryFile))
{
var summaryFileDirectory = Path.GetDirectoryName(destinationSummaryFile);
if (Directory.Exists(summaryFileDirectory))
{
Directory.Delete(path: summaryFileDirectory, recursive: true);
}
}
if (!Directory.Exists(_reportDirectory))
{
if (Directory.Exists(newReportDirectory))
{
//delete the generated report directory
Directory.Delete(path: newReportDirectory, recursive: true);
}
}
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Capabilities.
_term.WriteLine(StringUtil.Loc("ScanToolCapabilities"));
Dictionary <string, string> systemCapabilities = await HostContext.GetService <ICapabilitiesManager>().GetCapabilitiesAsync(_settings, token);
// Create connection.
Trace.Info("Loading Credentials");
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials creds = credMgr.LoadCredentials();
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = BuildConstants.AgentPackage.Version,
OSDescription = RuntimeInformation.OSDescription,
};
string sessionName = $"{Environment.MachineName ?? "AGENT"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent, systemCapabilities);
string errorMessage = string.Empty;
bool encounteringError = false;
_term.WriteLine(StringUtil.Loc("ConnectToServer"));
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Agent Server...");
await _agentServer.ConnectAsync(new Uri(serverUrl), creds);
Trace.Info("VssConnection created");
_session = await _agentServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine(StringUtil.Loc("QueueConnected", DateTime.UtcNow));
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Agent OAuth token has been revoked. Session creation failed.");
throw;
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, serverUrl, Trace.Error);
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
_term.WriteError(StringUtil.Loc("SessionCreateFailed", ex.Message));
return(false);
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError(StringUtil.Loc("QueueConError", DateTime.UtcNow, ex.Message, _sessionCreationRetryInterval.TotalSeconds));
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
// Refresh connection is best effort. it should never throw exception
public async Task RefreshConnectionAsync(AgentConnectionType connectionType, TimeSpan timeout)
{
Trace.Info($"Refresh {connectionType} VssConnection to get on a different AFD node.");
VssConnection newConnection = null;
switch (connectionType)
{
case AgentConnectionType.MessageQueue:
try
{
_hasMessageConnection = false;
newConnection = await EstablishVssConnection(_messageConnection.Uri, _messageConnection.Credentials, timeout);
var client = newConnection.GetClient <TaskAgentHttpClient>();
_messageConnection = newConnection;
_messageTaskAgentClient = client;
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, _requestConnection.Uri.ToString(), Trace.Error);
newConnection?.Dispose();
}
catch (Exception ex)
{
Trace.Error($"Catch exception during reset {connectionType} connection.");
Trace.Error(ex);
newConnection?.Dispose();
}
finally
{
_hasMessageConnection = true;
}
break;
case AgentConnectionType.JobRequest:
try
{
_hasRequestConnection = false;
newConnection = await EstablishVssConnection(_requestConnection.Uri, _requestConnection.Credentials, timeout);
var client = newConnection.GetClient <TaskAgentHttpClient>();
_requestConnection = newConnection;
_requestTaskAgentClient = client;
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, _requestConnection.Uri.ToString(), Trace.Error);
newConnection?.Dispose();
}
catch (Exception ex)
{
Trace.Error($"Catch exception during reset {connectionType} connection.");
Trace.Error(ex);
newConnection?.Dispose();
}
finally
{
_hasRequestConnection = true;
}
break;
case AgentConnectionType.Generic:
try
{
_hasGenericConnection = false;
newConnection = await EstablishVssConnection(_genericConnection.Uri, _genericConnection.Credentials, timeout);
var client = newConnection.GetClient <TaskAgentHttpClient>();
_genericConnection = newConnection;
_genericTaskAgentClient = client;
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, _requestConnection.Uri.ToString(), Trace.Error);
newConnection?.Dispose();
}
catch (Exception ex)
{
Trace.Error($"Catch exception during reset {connectionType} connection.");
Trace.Error(ex);
newConnection?.Dispose();
}
finally
{
_hasGenericConnection = true;
}
break;
default:
Trace.Error($"Unexpected connection type: {connectionType}.");
break;
}
}
public bool TryProcessCommand(IExecutionContext context, string input)
{
ArgUtil.NotNull(context, nameof(context));
if (string.IsNullOrEmpty(input))
{
return(false);
}
// TryParse input to Command
Command command;
var unescapePercents = AgentKnobs.DecodePercents.GetValue(context).AsBoolean();
if (!Command.TryParse(input, unescapePercents, out command))
{
// if parse fail but input contains ##vso, print warning with DOC link
if (input.IndexOf("##vso") >= 0)
{
context.Warning(StringUtil.Loc("CommandKeywordDetected", input));
}
return(false);
}
IWorkerCommandExtension extension = null;
if (_invokePluginInternalCommand && string.Equals(command.Area, _pluginInternalCommandExtensions.CommandArea, StringComparison.OrdinalIgnoreCase))
{
extension = _pluginInternalCommandExtensions;
}
if (extension != null || _commandExtensions.TryGetValue(command.Area, out extension))
{
if (!extension.SupportedHostTypes.HasFlag(context.Variables.System_HostType))
{
context.Error(StringUtil.Loc("CommandNotSupported", command.Area, context.Variables.System_HostType));
context.CommandResult = TaskResult.Failed;
return(false);
}
// process logging command in serialize oreder.
lock (_commandSerializeLock)
{
try
{
extension.ProcessCommand(context, command);
}
catch (SocketException ex)
{
#pragma warning disable CA2000 // Dispose objects before losing scope
ExceptionsUtil.HandleSocketException(ex, WorkerUtilities.GetVssConnection(context).Uri.ToString(), context.Error);
#pragma warning restore CA2000 // Dispose objects before losing scope
context.CommandResult = TaskResult.Failed;
}
catch (Exception ex)
{
context.Error(StringUtil.Loc("CommandProcessFailed", input));
context.Error(ex);
context.CommandResult = TaskResult.Failed;
}
finally
{
// trace the ##vso command as long as the command is not a ##vso[task.debug] command.
if (!(string.Equals(command.Area, "task", StringComparison.OrdinalIgnoreCase) &&
string.Equals(command.Event, "debug", StringComparison.OrdinalIgnoreCase)))
{
context.Debug($"Processed: {input}");
}
}
}
}
else
{
context.Warning(StringUtil.Loc("CommandNotFound", command.Area));
}
// Only if we've successfully parsed do we show this warning
if (AgentKnobs.DecodePercents.GetValue(context).AsString() == "" && input.Contains("%AZP25"))
{
context.Warning("%AZP25 detected in ##vso command. In March 2021, the agent command parser will be updated to unescape this to %. To opt out of this behavior, set a job level variable DECODE_PERCENTS to false. Setting to true will force this behavior immediately. More information can be found at https://github.com/microsoft/azure-pipelines-agent/blob/master/docs/design/percentEncoding.md");
}
return(true);
}