public virtual void Initialize(IDependencyResolver resolver, HostContext context)
{
if (resolver == null)
{
throw new ArgumentNullException("resolver");
}
if (context == null)
{
throw new ArgumentNullException("context");
}
if (_initialized)
{
return;
}
MessageBus = resolver.Resolve<IMessageBus>();
JsonSerializer = resolver.Resolve<IJsonSerializer>();
TraceManager = resolver.Resolve<ITraceManager>();
Counters = resolver.Resolve<IPerformanceCounterManager>();
AckHandler = resolver.Resolve<IAckHandler>();
ProtectedData = resolver.Resolve<IProtectedData>();
_configurationManager = resolver.Resolve<IConfigurationManager>();
_transportManager = resolver.Resolve<ITransportManager>();
_serverMessageHandler = resolver.Resolve<IServerCommandHandler>();
_initialized = true;
}
public override Task ProcessRequestAsync(HttpContextBase context)
#endif
{
var request = new AspNetRequest(context.Request, context.User);
var response = new AspNetResponse(context);
var hostContext = new HostContext(request, response);
#if NET45
// Determine if the client should bother to try a websocket request
hostContext.Items[HostConstants.SupportsWebSockets] = true;
#endif
// Set the debugging flag
hostContext.Items[HostConstants.DebugMode] = context.IsDebuggingEnabled;
// Set the host shutdown token
hostContext.Items[HostConstants.ShutdownToken] = AppDomainTokenSource.Token;
// Stick the context in here so transports or other asp.net specific logic can
// grab at it.
hostContext.Items["System.Web.HttpContext"] = context;
// Initialize the connection
_connection.Initialize(_resolver);
return _connection.ProcessRequestAsync(hostContext);
}
public void Setup([CallerMemberName] string testName = "")
{
_tokenSource = new CancellationTokenSource();
_hc = new HostContext(
hostType: "L0Test",
logFile: Path.Combine(IOUtil.GetBinPath(), $"trace_{nameof(HostContextL0)}_{testName}.log"));
}
protected TransportDisconnectBase(HostContext context, IJsonSerializer jsonSerializer, ITransportHeartbeat heartbeat, IPerformanceCounterManager performanceCounterManager, ITraceManager traceManager)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
if (jsonSerializer == null)
{
throw new ArgumentNullException("jsonSerializer");
}
if (heartbeat == null)
{
throw new ArgumentNullException("heartbeat");
}
if (performanceCounterManager == null)
{
throw new ArgumentNullException("performanceCounterManager");
}
if (traceManager == null)
{
throw new ArgumentNullException("traceManager");
}
_context = context;
_jsonSerializer = jsonSerializer;
_heartbeat = heartbeat;
_counters = performanceCounterManager;
_trace = traceManager["SignalR.Transports." + GetType().Name];
}
public HubContext(HostContext context, string connectionId)
{
ConnectionId = connectionId;
Cookies = context.Request.Cookies;
Headers = context.Request.Headers;
User = context.User;
}
public TransportDisconnectBase(HostContext context, IJsonSerializer jsonSerializer, ITransportHeartBeat heartBeat, IPerformanceCounterManager performanceCounterManager)
{
_context = context;
_jsonSerializer = jsonSerializer;
_heartBeat = heartBeat;
_counters = performanceCounterManager;
}
public void Execute(HostContext host, string[] parameters)
{
Console.WriteLine("System info:");
Console.WriteLine("----------------");
Console.WriteLine($"OS: {Environment.OSVersion}");
Console.WriteLine($".NET {Environment.Version}");
}
public override Task ProcessRequestAsync(HttpContextBase context)
#endif
{
// https://developer.mozilla.org/En/HTTP_Access_Control
string origin = context.Request.Headers["Origin"];
if (!String.IsNullOrEmpty(origin))
{
context.Response.AddHeader("Access-Control-Allow-Origin", origin);
context.Response.AddHeader("Access-Control-Allow-Credentials", "true");
}
var request = new AspNetRequest(context);
var response = new AspNetResponse(context);
var hostContext = new HostContext(request, response);
// Determine if the client should bother to try a websocket request
hostContext.Items[HostConstants.SupportsWebSockets] = !String.IsNullOrEmpty(context.Request.ServerVariables[WebSocketVersionServerVariable]);
// Set the debugging flag
hostContext.Items[HostConstants.DebugMode] = context.IsDebuggingEnabled;
// Set the host shutdown token
hostContext.Items[HostConstants.ShutdownToken] = AppDomainTokenSource.Token;
// Stick the context in here so transports or other asp.net specific logic can
// grab at it.
hostContext.Items["System.Web.HttpContext"] = context;
// Initialize the connection
_connection.Initialize(_resolver);
return _connection.ProcessRequestAsync(hostContext);
}
public ForeverTransport(HostContext context, IDependencyResolver resolver)
: this(context,
resolver.Resolve<IJsonSerializer>(),
resolver.Resolve<ITransportHeartBeat>(),
resolver.Resolve<IPerformanceCounterManager>())
{
}
private Task ProcessJsonpRequest(HostContext context, object payload)
{
context.Response.ContentType = JsonUtility.JavaScriptMimeType;
var data = JsonUtility.CreateJsonpCallback(context.Request.QueryString["callback"], JsonSerializer.Stringify(payload));
return context.Response.End(data);
}
public void SendUrlTriggersReceivedEvent()
{
var tcs = new TaskCompletionSource<string>();
var request = new Mock<IRequest>();
var form = new NameValueCollection();
form["data"] = "This is my data";
var qs = new NameValueCollection();
qs["connectionId"] = "1";
request.Setup(m => m.QueryString).Returns(qs);
request.Setup(m => m.Form).Returns(form);
request.Setup(m => m.Url).Returns(new Uri("http://test/echo/send"));
var counters = new Mock<IPerformanceCounterManager>();
var heartBeat = new Mock<ITransportHeartbeat>();
var json = new JsonNetSerializer();
var hostContext = new HostContext(request.Object, null);
var transportConnection = new Mock<ITransportConnection>();
var traceManager = new Mock<ITraceManager>();
traceManager.Setup(m => m[It.IsAny<string>()]).Returns(new System.Diagnostics.TraceSource("foo"));
var transport = new Mock<ForeverTransport>(hostContext, json, heartBeat.Object, counters.Object, traceManager.Object)
{
CallBase = true
};
transport.Object.Received = data =>
{
tcs.TrySetResult(data);
return TaskAsyncHelper.Empty;
};
transport.Object.ProcessRequest(transportConnection.Object).Wait();
Assert.Equal("This is my data", tcs.Task.Result);
}
public static void Main(string[] args)
{
RegisterCommands();
while (true)
{
var userLine = Console.ReadLine();
if (userLine == "Exit")
{
return;
}
var commandParts = userLine.Split();
var commandName = commandParts[0];
if (!commands.ContainsKey(commandName))
{
Console.WriteLine("Invalid command!");
continue;
}
var command = commands[commandName];
var context = new HostContext();
command.Execute(context, commandParts.Skip(1).ToArray());
}
}
public LongPollingTransport(HostContext context, IDependencyResolver resolver)
: this(context,
resolver.Resolve<IJsonSerializer>(),
resolver.Resolve<ITransportHeartBeat>(),
resolver.Resolve<IPerformanceCounterWriter>())
{
}
public void AbortUrlTriggersConnectionAbort()
{
var request = new Mock<IRequest>();
var qs = new NameValueCollection();
qs["connectionId"] = "1";
request.Setup(m => m.QueryString).Returns(qs);
request.Setup(m => m.Url).Returns(new Uri("http://test/echo/abort"));
string abortedConnectionId = null;
var counters = new Mock<IPerformanceCounterManager>();
var heartBeat = new Mock<ITransportHeartbeat>();
var json = new JsonNetSerializer();
var hostContext = new HostContext(request.Object, null);
var transportConnection = new Mock<ITransportConnection>();
var traceManager = new Mock<ITraceManager>();
traceManager.Setup(m => m[It.IsAny<string>()]).Returns(new System.Diagnostics.TraceSource("foo"));
transportConnection.Setup(m => m.Send(It.IsAny<ConnectionMessage>()))
.Callback<ConnectionMessage>(m =>
{
abortedConnectionId = m.Signal;
var command = m.Value as Command;
Assert.NotNull(command);
Assert.Equal(CommandType.Abort, command.CommandType);
})
.Returns(TaskAsyncHelper.Empty);
var transport = new Mock<ForeverTransport>(hostContext, json, heartBeat.Object, counters.Object, traceManager.Object)
{
CallBase = true
};
transport.Object.ProcessRequest(transportConnection.Object).Wait();
Assert.Equal("1", abortedConnectionId);
}
public Task Invoke(IDictionary<string, object> environment)
{
var serverRequest = new ServerRequest(environment);
var serverResponse = new ServerResponse(environment);
var hostContext = new HostContext(serverRequest, serverResponse);
// Add CORS support
var origins = serverRequest.RequestHeaders.GetHeaders("Origin");
if (origins != null && origins.Any(origin => !String.IsNullOrEmpty(origin)))
{
serverResponse.ResponseHeaders["Access-Control-Allow-Origin"] = origins;
serverResponse.ResponseHeaders["Access-Control-Allow-Credentials"] = AllowCredentialsTrue;
}
hostContext.Items[HostConstants.SupportsWebSockets] = LazyInitializer.EnsureInitialized(
ref _supportWebSockets,
ref _supportWebSocketsInitialized,
ref _supportWebSocketsLock,
() => environment.SupportsWebSockets());
hostContext.Items[HostConstants.ShutdownToken] = environment.GetShutdownToken();
hostContext.Items[HostConstants.DebugMode] = environment.GetIsDebugEnabled();
serverRequest.DisableRequestBuffering();
serverResponse.DisableResponseBuffering();
_connection.Initialize(_resolver, hostContext);
return _connection.ProcessRequestAsync(hostContext);
}
public WebSocketTransport(HostContext context,
IJsonSerializer serializer,
ITransportHeartBeat heartBeat)
: base(context, serializer, heartBeat)
{
_context = context;
}
public WebSocketTransport(HostContext context,
IDependencyResolver resolver)
: this(context,
resolver.Resolve<IJsonSerializer>(),
resolver.Resolve<ITransportHeartBeat>())
{
}
public WebSocketTransport(HostContext context,
IJsonSerializer serializer,
ITransportHeartBeat heartBeat,
IPerformanceCounterManager performanceCounterWriter)
: base(context, serializer, heartBeat, performanceCounterWriter)
{
_context = context;
}
public TransportDisconnectBase(HostContext context, IJsonSerializer jsonSerializer, ITransportHeartBeat heartBeat)
{
_context = context;
_jsonSerializer = jsonSerializer;
_heartBeat = heartBeat;
_timeoutTokenSource = new CancellationTokenSource();
_hostShutdownToken = context.HostShutdownToken();
}
public LongPollingTransport(HostContext context, IJsonSerializer jsonSerializer, ITransportHeartBeat heartBeat, IPerformanceCounterWriter performanceCounterWriter)
: base(context, jsonSerializer, heartBeat, performanceCounterWriter)
{
_jsonSerializer = jsonSerializer;
var counters = performanceCounterWriter;
_connConnectedCounter = counters.GetCounter(PerformanceCounters.ConnectionsConnected);
_connReconnectedCounter = counters.GetCounter(PerformanceCounters.ConnectionsReconnected);
}
public void Execute(HostContext host, string[] parameters)
{
Console.WriteLine(Environment.CurrentDirectory);
Console.WriteLine("---------------------");
foreach (var file in Directory.GetFiles(Environment.CurrentDirectory))
{
Console.WriteLine(new FileInfo(file).Name);
}
}
public ForeverTransport(HostContext context,
IJsonSerializer jsonSerializer,
ITransportHeartBeat heartBeat,
IPerformanceCounterManager performanceCounterWriter)
: base(context, jsonSerializer, heartBeat, performanceCounterWriter)
{
_jsonSerializer = jsonSerializer;
_counters = performanceCounterWriter;
}
public LongPollingTransport(HostContext context,
IJsonSerializer jsonSerializer,
ITransportHeartbeat heartbeat,
IPerformanceCounterManager performanceCounterManager,
ITraceManager traceManager)
: base(context, jsonSerializer, heartbeat, performanceCounterManager, traceManager)
{
_jsonSerializer = jsonSerializer;
_counters = performanceCounterManager;
}
public virtual Task ProcessRequestAsync(HostContext context)
{
if (!_initialized)
{
throw new InvalidOperationException("Connection not initialized.");
}
if (IsNegotiationRequest(context.Request))
{
return ProcessNegotiationRequest(context);
}
_transport = GetTransport(context);
if (_transport == null)
{
throw new InvalidOperationException("Unknown transport.");
}
string connectionId = _transport.ConnectionId;
// If there's no connection id then this is a bad request
if (String.IsNullOrEmpty(connectionId))
{
throw new InvalidOperationException("Protocol error: Missing connection id.");
}
var groups = new List<string>(_transport.Groups);
Connection = CreateConnection(connectionId, groups, context.Request);
_transport.Connected = () =>
{
return OnConnectedAsync(context.Request, groups, connectionId);
};
_transport.Reconnected = () =>
{
return OnReconnectedAsync(context.Request, groups, connectionId);
};
_transport.Received = data =>
{
return OnReceivedAsync(connectionId, data);
};
_transport.Error = OnErrorAsync;
_transport.Disconnected = () =>
{
return OnDisconnectAsync(connectionId);
};
return _transport.ProcessRequest(Connection) ?? TaskAsyncHelper.Empty;
}
public override void Initialize(IDependencyResolver resolver, HostContext context)
{
lock (lockobj)
{
if (timer==null)
{
timer = new Timer(scavageDictionary,null,TimeSpan.FromSeconds(60),TimeSpan.FromSeconds(60));
}
}
base.Initialize(resolver, context);
}
public void ForeverFrameTransportEscapesTags()
{
var request = new Mock<IRequest>();
var response = new CustomResponse();
var context = new HostContext(request.Object, response);
var fft = new ForeverFrameTransport(context, new DefaultDependencyResolver());
AssertEscaped(fft, response, "</sCRiPT>", "\\u003c/sCRiPT\\u003e");
AssertEscaped(fft, response, "</SCRIPT dosomething='false'>", "\\u003c/SCRIPT dosomething='false'\\u003e");
AssertEscaped(fft, response, "<p>ELLO</p>", "\\u003cp\\u003eELLO\\u003c/p\\u003e");
}
public TransportDisconnectBase(HostContext context, IJsonSerializer jsonSerializer, ITransportHeartBeat heartBeat)
{
_context = context;
_jsonSerializer = jsonSerializer;
_heartBeat = heartBeat;
_timeoutTokenSource = new CancellationTokenSource();
_hostShutdownToken = context.HostShutdownToken();
// Create a token that represents the end of this connection's life
_connectionEndToken = CancellationTokenSource.CreateLinkedTokenSource(_timeoutTokenSource.Token, _hostShutdownToken);
}
public void AvoidDeadlockIfCancellationTokenTriggeredBeforeSubscribing()
{
var request = new Mock<IRequest>();
var qs = new NameValueCollection();
qs["connectionId"] = "1";
request.Setup(m => m.QueryString).Returns(qs);
request.Setup(m => m.Url).Returns(new Uri("http://test/echo/connect"));
var counters = new Mock<IPerformanceCounterManager>();
var heartBeat = new Mock<ITransportHeartbeat>();
var json = new JsonNetSerializer();
var hostContext = new HostContext(request.Object, null);
var transportConnection = new Mock<ITransportConnection>();
var traceManager = new Mock<ITraceManager>();
traceManager.Setup(m => m[It.IsAny<string>()]).Returns(new System.Diagnostics.TraceSource("foo"));
Func<PersistentResponse, Task<bool>> callback = null;
transportConnection.Setup(m => m.Receive(It.IsAny<string>(),
It.IsAny<Func<PersistentResponse, Task<bool>>>(),
It.IsAny<int>())).Callback<string, Func<PersistentResponse, Task<bool>>, int>((id, cb, max) =>
{
callback = cb;
})
.Returns(new DisposableAction(() =>
{
callback(new PersistentResponse());
}));
var transport = new Mock<ForeverTransport>(hostContext, json, heartBeat.Object, counters.Object, traceManager.Object)
{
CallBase = true
};
var wh = new ManualResetEventSlim();
transport.Object.BeforeCancellationTokenCallbackRegistered = () =>
{
// Trip the cancellation token
transport.Object.End();
};
// Act
Task.Factory.StartNew(() =>
{
transport.Object.ProcessRequest(transportConnection.Object);
wh.Set();
});
Assert.True(wh.Wait(TimeSpan.FromSeconds(2)), "Dead lock!");
}
public override Task ProcessRequest(HostContext context)
{
var response = new Dictionary<string, object>();
response["Contracts"] = _contractGenerator != null ? _contractGenerator.GenerateContracts() : null;
if (!String.IsNullOrEmpty(context.Request.QueryString["callback"]))
{
return ProcessJsonpRequest(context, response);
}
context.Response.ContentType = JsonUtility.JsonMimeType;
return context.Response.End(JsonSerializer.Stringify(response));
}
public TransportDisconnectBase(HostContext context, IJsonSerializer jsonSerializer, ITransportHeartBeat heartBeat, IPerformanceCounterManager performanceCounterManager)
{
_context = context;
_jsonSerializer = jsonSerializer;
_heartBeat = heartBeat;
_timeoutTokenSource = new CancellationTokenSource();
_endTokenSource = new CancellationTokenSource();
_disconnectedToken = new CancellationTokenSource();
_hostShutdownToken = context.HostShutdownToken();
_counters = performanceCounterManager;
Completed = new TaskCompletionSource<object>();
// Create a token that represents the end of this connection's life
_connectionEndToken = CancellationTokenSource.CreateLinkedTokenSource(_timeoutTokenSource.Token, _endTokenSource.Token, _disconnectedToken.Token, _hostShutdownToken);
}
public void InitializeJob(JobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection));
ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = ApiUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Environment.Endpoints;
Endpoints.Add(message.Environment.SystemConnection);
// SecureFiles
SecureFiles = message.Environment.SecureFiles;
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings);
// Prepend Path
PrependPath = new List <string>();
// Docker
Container = new ContainerInfo()
{
ContainerImage = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"),
ContainerName = $"VSTS_{Variables.System_HostType.ToString()}_{message.JobId.ToString("D")}",
};
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobName,
refName: message.JobRefName,
order: 1); // The job timeline record must be at order 1.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_terminal = HostContext.GetService <ITerminal>();
}
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
foreach (var port in container.UserPortMappings)
{
Trace.Info($"User provided port: {port.Value}");
}
foreach (var volume in container.UserMountVolumes)
{
Trace.Info($"User provided volume: {volume.Value}");
}
if (container.ImageOS != PlatformUtil.OS.Windows)
{
string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory);
defaultWorkingDirectory = container.TranslateContainerPathForImageOS(PlatformUtil.HostOS, container.TranslateToContainerPath(defaultWorkingDirectory));
if (string.IsNullOrEmpty(defaultWorkingDirectory))
{
throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir"));
}
string workingDirectory = IOUtil.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), container.ImageOS);
string mountWorkingDirectory = container.TranslateToHostPath(workingDirectory);
executionContext.Debug($"Default Working Directory {defaultWorkingDirectory}");
executionContext.Debug($"Working Directory {workingDirectory}");
executionContext.Debug($"Mount Working Directory {mountWorkingDirectory}");
if (!string.IsNullOrEmpty(workingDirectory))
{
container.MountVolumes.Add(new MountVolume(mountWorkingDirectory, workingDirectory));
}
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks))));
}
else
{
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
}
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
bool externalReadOnly = container.ImageOS != PlatformUtil.OS.Windows; // This code was refactored to use PlatformUtils. The previous implementation did not have the externals directory mounted read-only for Windows.
// That seems wrong, but to prevent any potential backwards compatibility issues, we are keeping the same logic
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), externalReadOnly));
if (container.ImageOS != PlatformUtil.OS.Windows)
{
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
}
if (container.IsJobContainer)
{
// See if this container brings its own Node.js
container.CustomNodePath = await _dockerManger.DockerInspect(context : executionContext,
dockerObject : container.ContainerImage,
options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\"");
string node;
if (!string.IsNullOrEmpty(container.CustomNodePath))
{
node = container.CustomNodePath;
}
else
{
node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
// if on Mac OS X, require container to have node
if (PlatformUtil.RunningOnMacOS)
{
container.CustomNodePath = "node";
node = container.CustomNodePath;
}
// if running on Windows, and attempting to run linux container, require container to have node
else if (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux)
{
container.CustomNodePath = "node";
node = container.CustomNodePath;
}
}
string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
container.ContainerCommand = sleepCommand;
}
container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
if (container.IsJobContainer)
{
executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId);
}
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
// Get port mappings of running container
if (!container.IsJobContainer)
{
container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId));
foreach (var port in container.PortMappings)
{
executionContext.Variables.Set(
$"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}",
$"{port.HostPort}");
}
}
if (!PlatformUtil.RunningOnWindows)
{
if (container.IsJobContainer)
{
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\"");
if (execWhichBashExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Get current username
container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName));
// Get current userId
container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId));
executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId));
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
string containerUserName = string.Empty;
// We need to find out whether there is a user with same UID inside the container
List <string> userNames = new List <string>();
int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"getent passwd {container.CurrentUserId} | cut -d: -f1 \"", userNames);
if (execGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}");
}
if (userNames.Count > 0)
{
// check all potential username that might match the UID.
foreach (string username in userNames)
{
int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}");
if (execIdExitCode == 0)
{
containerUserName = username;
break;
}
}
}
// Create a new user with same UID
if (string.IsNullOrEmpty(containerUserName))
{
containerUserName = $"{container.CurrentUserName}_azpcontainer";
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName));
// Create a new group for giving sudo permission
int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo");
if (execGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}");
}
// Add the new created user to the new created sudo group.
int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}");
}
// Allow the new sudo group run any sudo command without providing password.
int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\"");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}");
}
if (AgentKnobs.SetupDockerGroup.GetValue(executionContext).AsBoolean())
{
executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName));
// Get docker.sock group id on Host
string statFormatOption = "-c %g";
if (PlatformUtil.RunningOnMacOS)
{
statFormatOption = "-f %g";
}
string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"{statFormatOption} /var/run/docker.sock")).FirstOrDefault();
// We need to find out whether there is a group with same GID inside the container
string existingGroupName = null;
List <string> groupsOutput = new List <string>();
int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput);
if (execGroupGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}");
}
if (groupsOutput.Count > 0)
{
// check all potential groups that might match the GID.
foreach (string groupOutput in groupsOutput)
{
if (!string.IsNullOrEmpty(groupOutput))
{
var groupSegments = groupOutput.Split(':');
if (groupSegments.Length != 4)
{
Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'");
}
else
{
// the output of /etc/group should looks like `group:x:gid:`
var groupName = groupSegments[0];
var groupId = groupSegments[2];
if (string.Equals(dockerSockGroupId, groupId))
{
existingGroupName = groupName;
break;
}
}
}
}
}
if (string.IsNullOrEmpty(existingGroupName))
{
// create a new group with same gid
existingGroupName = "azure_pipelines_docker";
int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker");
if (execDockerGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}");
}
}
// Add the new created user to the docker socket group.
int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}");
if (execGroupUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}");
}
// if path to node is just 'node', with no path, let's make sure it is actually there
if (string.Equals(container.CustomNodePath, "node", StringComparison.OrdinalIgnoreCase))
{
List <string> nodeVersionOutput = new List <string>();
int execNodeVersionExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"node -v\"", nodeVersionOutput);
if (execNodeVersionExitCode != 0)
{
throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. Got exit code {execNodeVersionExitCode} from docker exec");
}
if (nodeVersionOutput.Count > 0)
{
executionContext.Output($"Detected Node Version: {nodeVersionOutput[0]}");
Trace.Info($"Using node version {nodeVersionOutput[0]} in container {container.ContainerId}");
}
else
{
throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. No output from node -v");
}
}
}
}
}
}
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
}
private void AddProxySetting(IVstsAgentWebProxy agentProxy)
{
string appConfig = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), _appConfigFileName);
ArgUtil.File(appConfig, _appConfigFileName);
string appConfigRestore = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), _appConfigRestoreFileName);
if (!File.Exists(appConfigRestore))
{
Trace.Info("Take snapshot of current appconfig for restore modified appconfig.");
File.Copy(appConfig, appConfigRestore);
}
else
{
// cleanup any appconfig changes from previous build.
ExecutionContext.Debug("Restore default LegacyVSTSPowerShellHost.exe.config.");
IOUtil.DeleteFile(appConfig);
File.Copy(appConfigRestore, appConfig);
}
XmlDocument psHostAppConfig = new XmlDocument();
using (var appConfigStream = new FileStream(appConfig, FileMode.Open, FileAccess.Read))
{
psHostAppConfig.Load(appConfigStream);
}
var configuration = psHostAppConfig.SelectSingleNode("configuration");
ArgUtil.NotNull(configuration, "configuration");
var exist_defaultProxy = psHostAppConfig.SelectSingleNode("configuration/system.net/defaultProxy");
if (exist_defaultProxy == null)
{
var system_net = psHostAppConfig.SelectSingleNode("configuration/system.net");
if (system_net == null)
{
Trace.Verbose("Create system.net section in appconfg.");
system_net = psHostAppConfig.CreateElement("system.net");
}
Trace.Verbose("Create defaultProxy section in appconfg.");
var defaultProxy = psHostAppConfig.CreateElement("defaultProxy");
defaultProxy.SetAttribute("useDefaultCredentials", "true");
Trace.Verbose("Create proxy section in appconfg.");
var proxy = psHostAppConfig.CreateElement("proxy");
proxy.SetAttribute("proxyaddress", agentProxy.ProxyAddress);
proxy.SetAttribute("bypassonlocal", "true");
if (agentProxy.ProxyBypassList != null && agentProxy.ProxyBypassList.Count > 0)
{
Trace.Verbose("Create bypasslist section in appconfg.");
var bypass = psHostAppConfig.CreateElement("bypasslist");
foreach (string proxyBypass in agentProxy.ProxyBypassList)
{
Trace.Verbose($"Create bypasslist.add section for {proxyBypass} in appconfg.");
var add = psHostAppConfig.CreateElement("add");
add.SetAttribute("address", proxyBypass);
bypass.AppendChild(add);
}
defaultProxy.AppendChild(bypass);
}
defaultProxy.AppendChild(proxy);
system_net.AppendChild(defaultProxy);
configuration.AppendChild(system_net);
using (var appConfigStream = new FileStream(appConfig, FileMode.Open, FileAccess.ReadWrite))
{
psHostAppConfig.Save(appConfigStream);
}
ExecutionContext.Debug("Add Proxy setting in LegacyVSTSPowerShellHost.exe.config file.");
}
else
{
//proxy setting exist.
ExecutionContext.Debug("Proxy setting already exist in LegacyVSTSPowerShellHost.exe.config file.");
}
}
public override async Task ProcessRequestAsync(IRequest request, IResponse response, string operationName)
{
HostContext.ApplyCustomHandlerRequestFilters(request, response);
if (response.IsClosed)
{
return;
}
await response.EndHttpHandlerRequestAsync(afterHeaders : async r =>
{
var node = this.VirtualNode ?? request.GetVirtualNode();
var file = node as IVirtualFile;
if (file == null)
{
var dir = node as IVirtualDirectory;
if (dir != null)
{
file = dir.GetDefaultDocument(HostContext.AppHost.Config.DefaultDocuments);
if (file != null && HostContext.Config.RedirectToDefaultDocuments)
{
r.Redirect(request.GetPathUrl() + '/' + file.Name);
return;
}
}
if (file == null)
{
var fileName = request.PathInfo;
var originalFileName = fileName;
if (Env.IsMono)
{
//Create a case-insensitive file index of all host files
if (allFiles == null)
{
allFiles = CreateFileIndex(HostContext.VirtualFileSources.RootDirectory.RealPath);
}
if (allDirs == null)
{
allDirs = CreateDirIndex(HostContext.VirtualFileSources.RootDirectory.RealPath);
}
if (allFiles.TryGetValue(fileName.ToLower(), out fileName))
{
file = HostContext.VirtualFileSources.GetFile(fileName);
}
}
if (file == null)
{
var msg = ErrorMessages.FileNotExistsFmt.Fmt(request.PathInfo.SafeInput());
log.Warn($"{msg} in path: {originalFileName}");
response.StatusCode = 404;
response.StatusDescription = msg;
return;
}
}
}
file.Refresh(); //refresh FileInfo, DateModified, Length
TimeSpan maxAge;
if (r.ContentType != null && HostContext.Config.AddMaxAgeForStaticMimeTypes.TryGetValue(r.ContentType, out maxAge))
{
r.AddHeader(HttpHeaders.CacheControl, "max-age=" + maxAge.TotalSeconds);
}
if (request.HasNotModifiedSince(file.LastModified))
{
r.ContentType = MimeTypes.GetMimeType(file.Name);
r.StatusCode = (int)HttpStatusCode.NotModified;
r.StatusDescription = HttpStatusCode.NotModified.ToString();
return;
}
try
{
var encoding = request.GetCompressionType();
var shouldCompress = encoding != null && HostContext.AppHost.ShouldCompressFile(file);
r.AddHeaderLastModified(file.LastModified);
r.ContentType = MimeTypes.GetMimeType(file.Name);
if (ResponseFilter != null)
{
ResponseFilter(request, r, file);
if (r.IsClosed)
{
return;
}
}
if (!HostContext.DebugMode && file.VirtualPath.EqualsIgnoreCase(DefaultFilePath))
{
if (file.LastModified > DefaultFileModified)
{
SetDefaultFile(DefaultFilePath, file.ReadAllBytes(), file.LastModified); //reload
}
if (!shouldCompress)
{
await r.OutputStream.WriteAsync(DefaultFileContents);
await r.OutputStream.FlushAsync();
}
else
{
byte[] zipBytes = null;
if (encoding == CompressionTypes.GZip)
{
zipBytes = DefaultFileContentsGzip ??
(DefaultFileContentsGzip = DefaultFileContents.CompressBytes(encoding));
}
else if (encoding == CompressionTypes.Deflate)
{
zipBytes = DefaultFileContentsDeflate ??
(DefaultFileContentsDeflate = DefaultFileContents.CompressBytes(encoding));
}
else
{
zipBytes = DefaultFileContents.CompressBytes(encoding);
}
r.AddHeader(HttpHeaders.ContentEncoding, encoding);
r.SetContentLength(zipBytes.Length);
await r.OutputStream.WriteAsync(zipBytes);
await r.OutputStream.FlushAsync();
}
r.Close();
return;
}
if (HostContext.Config.AllowPartialResponses)
{
r.AddHeader(HttpHeaders.AcceptRanges, "bytes");
}
long contentLength = file.Length;
long rangeStart, rangeEnd;
var rangeHeader = request.Headers[HttpHeaders.Range];
if (HostContext.Config.AllowPartialResponses && rangeHeader != null)
{
rangeHeader.ExtractHttpRanges(contentLength, out rangeStart, out rangeEnd);
if (rangeEnd > contentLength - 1)
{
rangeEnd = contentLength - 1;
}
r.AddHttpRangeResponseHeaders(rangeStart: rangeStart, rangeEnd: rangeEnd, contentLength: contentLength);
}
else
{
rangeStart = 0;
rangeEnd = contentLength - 1;
}
var outputStream = r.OutputStream;
using (var fs = file.OpenRead())
{
if (rangeStart != 0 || rangeEnd != file.Length - 1)
{
await fs.WritePartialToAsync(outputStream, rangeStart, rangeEnd);
}
else
{
if (!shouldCompress)
{
r.SetContentLength(contentLength);
await fs.CopyToAsync(outputStream, BufferSize);
outputStream.Flush();
}
else
{
r.AddHeader(HttpHeaders.ContentEncoding, encoding);
outputStream = outputStream.CompressStream(encoding);
await fs.CopyToAsync(outputStream);
await outputStream.FlushAsync();
outputStream.Close();
}
}
}
}
#if !NETSTANDARD1_6
catch (System.Net.HttpListenerException ex)
{
if (ex.ErrorCode == 1229)
{
return;
}
//Error: 1229 is "An operation was attempted on a nonexistent network connection"
//This exception occures when http stream is terminated by web browser because user
//seek video forward and new http request will be sent by browser
//with attribute in header "Range: bytes=newSeekPosition-"
throw;
}
#endif
catch (Exception ex)
{
log.ErrorFormat($"Static file {request.PathInfo} forbidden: {ex.Message}");
throw new HttpException(403, "Forbidden.");
}
});
}
public IEndpoint ResolveCommandReplyToEndpoint <TResponse>(HostContext hostContext) where TResponse : IMessage
{
var queueName = $"{hostContext.ServiceName}.Responses";
return(this.ResolveCommandReplyToEndpoint(queueName));
}
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = ApiUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Resources.Endpoints;
// SecureFiles
SecureFiles = message.Resources.SecureFiles;
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Variables, out warnings);
// Prepend Path
PrependPath = new List <string>();
// Docker
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
if (string.IsNullOrEmpty(imageName))
{
imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE");
}
if (!string.IsNullOrEmpty(imageName) &&
string.IsNullOrEmpty(message.JobContainer))
{
var dockerContainer = new Pipelines.ContainerReference()
{
Name = "vsts_container_preview"
};
dockerContainer.Data["image"] = imageName;
Container = new ContainerInfo(dockerContainer);
}
else if (!string.IsNullOrEmpty(message.JobContainer))
{
Container = new ContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Name, message.JobContainer, StringComparison.OrdinalIgnoreCase)));
}
else
{
Container = null;
}
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
if (agentWebProxy.ProxyBypassList.Count > 0)
{
Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList));
}
}
// Certificate variables
var agentCert = HostContext.GetService <IAgentCertificateManager>();
if (agentCert.SkipServerCertificateValidation)
{
Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString);
}
if (!string.IsNullOrEmpty(agentCert.CACertificateFile))
{
Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile);
}
if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile))
{
Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile);
Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile);
Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile);
if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword))
{
Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true);
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobDisplayName,
refName: message.JobName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public override async Task <IHttpResult> OnAuthenticatedAsync(IServiceBase authService, IAuthSession session, IAuthTokens tokens, Dictionary <string, string> authInfo, CancellationToken token = default)
{
session.AuthProvider = Name;
if (session is AuthUserSession userSession)
{
await LoadUserAuthInfoAsync(userSession, tokens, authInfo, token).ConfigAwait();
HostContext.TryResolve <IAuthMetadataProvider>().SafeAddMetadata(tokens, authInfo);
}
if (session is IAuthSessionExtended authSession)
{
var failed = authSession.Validate(authService, session, tokens, authInfo)
?? await authSession.ValidateAsync(authService, session, tokens, authInfo, token)
?? AuthEvents.Validate(authService, session, tokens, authInfo)
?? (AuthEvents is IAuthEventsAsync asyncEvents
? await asyncEvents.ValidateAsync(authService, session, tokens, authInfo, token)
: null);
if (failed != null)
{
await authService.RemoveSessionAsync(token).ConfigAwait();
return(failed);
}
}
var authRepo = GetUserAuthRepositoryAsync(authService.Request);
#if NET472 || NETSTANDARD2_0
await using (authRepo as IAsyncDisposable)
#else
using (authRepo as IDisposable)
#endif
{
if (authRepo != null)
{
if (tokens != null)
{
authInfo.ForEach((x, y) => tokens.Items[x] = y);
session.UserAuthId = (await authRepo.CreateOrMergeAuthSessionAsync(session, tokens, token)).UserAuthId.ToString();
}
foreach (var oAuthToken in session.GetAuthTokens())
{
var authProvider = AuthenticateService.GetAuthProvider(oAuthToken.Provider);
var userAuthProvider = authProvider as OAuthProvider;
userAuthProvider?.LoadUserOAuthProvider(session, oAuthToken);
}
var failed = await ValidateAccountAsync(authService, authRepo, session, tokens, token).ConfigAwait();
if (failed != null)
{
return(failed);
}
}
}
try
{
session.IsAuthenticated = true;
session.OnAuthenticated(authService, session, tokens, authInfo);
if (session is IAuthSessionExtended sessionExt)
{
await sessionExt.OnAuthenticatedAsync(authService, session, tokens, authInfo, token).ConfigAwait();
}
AuthEvents.OnAuthenticated(authService.Request, session, authService, tokens, authInfo);
if (AuthEvents is IAuthEventsAsync asyncEvents)
{
await asyncEvents.OnAuthenticatedAsync(authService.Request, session, authService, tokens, authInfo, token).ConfigAwait();
}
}
finally
{
await this.SaveSessionAsync(authService, session, SessionExpiry, token).ConfigAwait();
authService.Request.Items[Keywords.DidAuthenticate] = true;
}
return(null);
}
public override void InitializeJobExtension(IExecutionContext executionContext, IList <JobStep> steps, WorkspaceOptions workspace)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
executionContext.Output(StringUtil.Loc("PrepareReleasesDir"));
var directoryManager = HostContext.GetService <IReleaseDirectoryManager>();
ReleaseId = executionContext.Variables.GetInt(Constants.Variables.Release.ReleaseId) ?? 0;
TeamProjectId = executionContext.Variables.GetGuid(Constants.Variables.System.TeamProjectId) ?? Guid.Empty;
SkipArtifactsDownload = executionContext.Variables.GetBoolean(Constants.Variables.Release.SkipArtifactsDownload) ?? false;
string releaseDefinitionName = executionContext.Variables.Get(Constants.Variables.Release.ReleaseDefinitionName);
// TODO: Should we also write to log in executionContext.Output methods? so that we don't have to repeat writing into logs?
// Log these values here to debug scenarios where downloading the artifact fails.
executionContext.Output($"ReleaseId={ReleaseId}, TeamProjectId={TeamProjectId}, ReleaseDefinitionName={releaseDefinitionName}");
var releaseDefinition = executionContext.Variables.Get(Constants.Variables.Release.ReleaseDefinitionId);
if (string.IsNullOrEmpty(releaseDefinition))
{
string pattern = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
Regex regex = new Regex(string.Format("[{0}]", Regex.Escape(pattern)));
releaseDefinition = regex.Replace(releaseDefinitionName, string.Empty);
}
var releaseTrackingConfig = directoryManager.PrepareArtifactsDirectory(
HostContext.GetDirectory(WellKnownDirectory.Work),
executionContext.Variables.System_CollectionId,
executionContext.Variables.System_TeamProjectId.ToString(),
releaseDefinition);
ReleaseWorkingFolder = releaseTrackingConfig.ReleaseDirectory;
ArtifactsWorkingFolder = Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.Work),
releaseTrackingConfig.ReleaseDirectory,
Constants.Release.Path.ArtifactsDirectory);
executionContext.Output($"Release folder: {ArtifactsWorkingFolder}");
// Ensure directory exist
if (!Directory.Exists(ArtifactsWorkingFolder))
{
Trace.Info($"Creating {ArtifactsWorkingFolder}.");
Directory.CreateDirectory(ArtifactsWorkingFolder);
}
SetLocalVariables(executionContext, ArtifactsWorkingFolder);
// Log the environment variables available after populating the variable service with our variables
LogEnvironmentVariables(executionContext);
if (SkipArtifactsDownload)
{
// If this is the first time the agent is executing a task, we need to create the artifactsFolder
// otherwise Process.StartWithCreateProcess() will fail with the error "The directory name is invalid"
// because the working folder doesn't exist
CreateWorkingFolderIfRequired(executionContext, ArtifactsWorkingFolder);
// log the message that the user chose to skip artifact download and move on
executionContext.Output(StringUtil.Loc("RMUserChoseToSkipArtifactDownload"));
Trace.Info("Skipping artifact download based on the setting specified.");
}
else
{
ReleaseArtifacts = GetReleaseArtifacts(executionContext);
if (!ReleaseArtifacts.Any())
{
CreateArtifactsFolder(executionContext, ArtifactsWorkingFolder);
Trace.Info("No artifacts found to be downloaded by agent.");
}
}
CheckForAvailableDiskSpace(executionContext);
}
public List <PostmanRequest> GetRequests(Postman request, string parentId, IEnumerable <Operation> operations)
{
var ret = new List <PostmanRequest>();
var feature = HostContext.GetPlugin <PostmanFeature>();
var headers = feature.Headers ?? ("Accept: " + MimeTypes.Json);
if (Response is IHttpResponse httpRes)
{
if (request.ssopt != null ||
request.sspid != null ||
request.ssid != null)
{
if (feature.EnableSessionExport != true)
{
throw new ArgumentException("PostmanFeature.EnableSessionExport is not enabled");
}
}
if (request.ssopt != null)
{
Request.AddSessionOptions(request.ssopt);
}
if (request.sspid != null)
{
httpRes.Cookies.AddPermanentCookie(SessionFeature.PermanentSessionId, request.sspid);
}
if (request.ssid != null)
{
httpRes.Cookies.AddSessionCookie(SessionFeature.SessionId, request.ssid,
(HostContext.Config.UseSecureCookies && Request.IsSecureConnection));
}
}
foreach (var op in operations)
{
Uri url = null;
if (!HostContext.Metadata.IsVisible(base.Request, op))
{
continue;
}
var allVerbs = new HashSet <string>(op.Actions.Concat(
op.Routes.SelectMany(x => x.Verbs))
.SelectMany(x => x == ActionContext.AnyAction
? feature.DefaultVerbsForAny
: new List <string> {
x
}));
var propertyTypes = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
op.RequestType.GetSerializableFields()
.Each(x => propertyTypes[x.Name] = x.FieldType.AsFriendlyName(feature));
op.RequestType.GetSerializableProperties()
.Each(x => propertyTypes[x.Name] = x.PropertyType.AsFriendlyName(feature));
foreach (var route in op.Routes)
{
var routeVerbs = route.Verbs.Contains(ActionContext.AnyAction)
? feature.DefaultVerbsForAny.ToArray()
: route.Verbs;
var restRoute = route.ToRestRoute();
foreach (var verb in routeVerbs)
{
allVerbs.Remove(verb); //exclude handled verbs
var routeData = restRoute.QueryStringVariables
.Map(x => new PostmanData
{
key = x,
value = "",
type = "text",
})
.ApplyPropertyTypes(propertyTypes);
url = new Uri(Request.GetBaseUrl().CombineWith(restRoute.Path.ToPostmanPathVariables()));
ret.Add(new PostmanRequest
{
request = new PostmanRequestDetails {
url = new PostmanRequestUrl {
raw = url.OriginalString,
host = url.Host,
port = url.Port.ToString(),
protocol = url.Scheme,
path = url.LocalPath.SplitPaths(),
query = !HttpUtils.HasRequestBody(verb)
? routeData.Select(x => x.key)
.ApplyPropertyTypes(propertyTypes)
.Map(x => new PostmanRequestKeyValue {
key = x.Key, value = x.Value
})
: null,
variable = restRoute.Variables.Any()
? restRoute.Variables.Map(x => new PostmanRequestKeyValue {
key = x
})
: null
},
method = verb,
body = new PostmanRequestBody {
formdata = HttpUtils.HasRequestBody(verb)
? routeData
: null,
},
header = headers,
},
name = GetName(feature, request, op.RequestType, restRoute.Path),
});
}
}
var emptyRequest = op.RequestType.CreateInstance();
var virtualPath = emptyRequest.ToReplyUrlOnly();
var requestParams = propertyTypes
.Map(x => new PostmanData
{
key = x.Key,
value = x.Value,
type = "text",
});
url = new Uri(Request.GetBaseUrl().CombineWith(virtualPath));
ret.AddRange(allVerbs.Select(verb =>
new PostmanRequest
{
request = new PostmanRequestDetails {
url = new PostmanRequestUrl {
raw = url.OriginalString,
host = url.Host,
port = url.Port.ToString(),
protocol = url.Scheme,
path = url.LocalPath.SplitPaths(),
query = !HttpUtils.HasRequestBody(verb)
? requestParams.Select(x => x.key)
.Where(x => !x.StartsWith(":"))
.ApplyPropertyTypes(propertyTypes)
.Map(x => new PostmanRequestKeyValue {
key = x.Key, value = x.Value
})
: null,
variable = url.Segments.Any(x => x.StartsWith(":"))
? url.Segments.Where(x => x.StartsWith(":"))
.Map(x => new PostmanRequestKeyValue {
key = x.Replace(":", ""), value = ""
})
: null
},
method = verb,
body = new PostmanRequestBody {
formdata = HttpUtils.HasRequestBody(verb)
? requestParams
: null,
},
header = headers,
},
name = GetName(feature, request, op.RequestType, virtualPath),
}));
}
return(ret);
}
private async Task DownloadArtifacts(IExecutionContext executionContext,
IList <AgentArtifactDefinition> agentArtifactDefinitions, string artifactsWorkingFolder)
{
Trace.Entering();
CreateArtifactsFolder(executionContext, artifactsWorkingFolder);
executionContext.Output(StringUtil.Loc("RMDownloadingArtifact"));
foreach (AgentArtifactDefinition agentArtifactDefinition in agentArtifactDefinitions)
{
// We don't need to check if its old style artifact anymore. All the build data has been fixed and all the build artifact has Alias now.
ArgUtil.NotNullOrEmpty(agentArtifactDefinition.Alias, nameof(agentArtifactDefinition.Alias));
var extensionManager = HostContext.GetService <IExtensionManager>();
IArtifactExtension extension =
(extensionManager.GetExtensions <IArtifactExtension>()).FirstOrDefault(x =>
agentArtifactDefinition.ArtifactType == x.ArtifactType);
if (extension == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMArtifactTypeNotSupported",
agentArtifactDefinition.ArtifactType));
}
Trace.Info($"Found artifact extension of type {extension.ArtifactType}");
executionContext.Output(StringUtil.Loc("RMStartArtifactsDownload"));
ArtifactDefinition artifactDefinition =
ConvertToArtifactDefinition(agentArtifactDefinition, executionContext, extension);
executionContext.Output(StringUtil.Loc("RMArtifactDownloadBegin", agentArtifactDefinition.Alias,
agentArtifactDefinition.ArtifactType));
// Get the local path where this artifact should be downloaded.
string downloadFolderPath = Path.GetFullPath(Path.Combine(artifactsWorkingFolder,
agentArtifactDefinition.Alias ?? string.Empty));
// download the artifact to this path.
RetryExecutor retryExecutor = new RetryExecutor();
retryExecutor.ShouldRetryAction = (ex) =>
{
executionContext.Output(StringUtil.Loc("RMErrorDuringArtifactDownload", ex));
bool retry = true;
if (ex is ArtifactDownloadException)
{
retry = false;
}
else
{
executionContext.Output(StringUtil.Loc("RMRetryingArtifactDownload"));
Trace.Warning(ex.ToString());
}
return(retry);
};
await retryExecutor.ExecuteAsync(
async() =>
{
var releaseFileSystemManager = HostContext.GetService <IReleaseFileSystemManager>();
executionContext.Output(StringUtil.Loc("RMEnsureArtifactFolderExistsAndIsClean",
downloadFolderPath));
releaseFileSystemManager.EnsureEmptyDirectory(downloadFolderPath,
executionContext.CancellationToken);
await extension.DownloadAsync(executionContext, artifactDefinition, downloadFolderPath);
});
executionContext.Output(StringUtil.Loc("RMArtifactDownloadFinished",
agentArtifactDefinition.Alias));
}
executionContext.Output(StringUtil.Loc("RMArtifactsDownloadFinished"));
}
private string GetBaseUrl(string baseUrl) => baseUrl ?? HostContext.GetPlugin <NativeTypesFeature>().MetadataTypesConfig.BaseUrl ?? Request.GetBaseUrl();
public override IHttpResult OnAuthenticated(IServiceBase authService, IAuthSession session, IAuthTokens tokens, Dictionary <string, string> authInfo)
{
var userSession = session as AuthUserSession;
if (userSession != null)
{
LoadUserAuthInfo(userSession, tokens, authInfo);
HostContext.TryResolve <IAuthMetadataProvider>().SafeAddMetadata(tokens, authInfo);
LoadUserAuthFilter?.Invoke(userSession, tokens, authInfo);
}
var authRepo = HostContext.AppHost.GetAuthRepository(authService.Request);
using (authRepo as IDisposable)
{
if (CustomValidationFilter != null)
{
var ctx = new AuthContext
{
Request = authService.Request,
Service = authService,
AuthProvider = this,
Session = session,
AuthTokens = tokens,
AuthInfo = authInfo,
AuthRepository = authRepo,
};
var response = CustomValidationFilter(ctx);
if (response != null)
{
authService.RemoveSession();
return(response);
}
}
if (authRepo != null)
{
if (tokens != null)
{
authInfo.ForEach((x, y) => tokens.Items[x] = y);
session.UserAuthId = authRepo.CreateOrMergeAuthSession(session, tokens).UserAuthId.ToString();
}
foreach (var oAuthToken in session.GetAuthTokens())
{
var authProvider = AuthenticateService.GetAuthProvider(oAuthToken.Provider);
var userAuthProvider = authProvider as OAuthProvider;
userAuthProvider?.LoadUserOAuthProvider(session, oAuthToken);
}
var httpRes = authService.Request.Response as IHttpResponse;
httpRes?.Cookies.AddPermanentCookie(HttpHeaders.XUserAuthId, session.UserAuthId);
var failed = ValidateAccount(authService, authRepo, session, tokens);
if (failed != null)
{
return(failed);
}
}
}
try
{
session.IsAuthenticated = true;
session.OnAuthenticated(authService, session, tokens, authInfo);
AuthEvents.OnAuthenticated(authService.Request, session, authService, tokens, authInfo);
}
finally
{
this.SaveSession(authService, session, SessionExpiry);
}
return(null);
}
public TrackingConfig PrepareDirectory(
IExecutionContext executionContext,
IList <RepositoryResource> repositories,
WorkspaceOptions workspace)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repositories, nameof(repositories));
var trackingManager = HostContext.GetService <ITrackingManager>();
// Create the tracking config for this execution of the pipeline
var agentSettings = HostContext.GetService <IConfigurationStore>().GetSettings();
var shouldOverrideBuildDirectory = ShouldOverrideBuildDirectory(repositories, agentSettings);
var newConfig = trackingManager.Create(executionContext, repositories, shouldOverrideBuildDirectory);
// Load the tracking config from the last execution of the pipeline
var existingConfig = trackingManager.LoadExistingTrackingConfig(executionContext);
// If there aren't any major changes, merge the configurations and use the same workspace
if (trackingManager.AreTrackingConfigsCompatible(executionContext, newConfig, existingConfig))
{
bool disableOverrideTfvcBuildDirectoryKnob = AgentKnobs.DisableOverrideTfvcBuildDirectory.GetValue(executionContext).AsBoolean();
newConfig = trackingManager.MergeTrackingConfigs(executionContext, newConfig, existingConfig, shouldOverrideBuildDirectory && !disableOverrideTfvcBuildDirectoryKnob);
}
else if (existingConfig != null)
{
// If the previous config had different repos, get a new workspace folder and mark the old one for clean up
trackingManager.MarkForGarbageCollection(executionContext, existingConfig);
// If the config file was updated to a new config, we need to delete the legacy artifact/staging directories.
// DeleteDirectory will check for the existence of the folders first.
DeleteDirectory(
executionContext,
description: "legacy artifacts directory",
path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyArtifactsDirectory));
DeleteDirectory(
executionContext,
description: "legacy staging directory",
path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyStagingDirectory));
}
// Save any changes to the config file
trackingManager.UpdateTrackingConfig(executionContext, newConfig);
// Prepare the build directory.
// There are 2 ways to provide build directory clean policy.
// 1> set definition variable build.clean or agent.clean.buildDirectory. (on-prem user need to use this, since there is no Web UI in TFS 2016)
// 2> select source clean option in definition repository tab. (VSTS will have this option in definition designer UI)
BuildCleanOption cleanOption = GetBuildDirectoryCleanOption(executionContext, workspace);
CreateDirectory(
executionContext,
description: "build directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory),
deleteExisting: cleanOption == BuildCleanOption.All);
CreateDirectory(
executionContext,
description: "artifacts directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.ArtifactsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "test results directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.TestResultsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "binaries directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Binary);
var defaultSourceDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory);
CreateDirectory(
executionContext,
description: "source directory",
path: defaultSourceDirectory,
deleteExisting: cleanOption == BuildCleanOption.Source);
// Set the default clone path for each repository (the Checkout task may override this later)
foreach (var repository in repositories)
{
string repoPath = GetDefaultRepositoryPath(executionContext, repository, newConfig);
if (!string.Equals(repoPath, defaultSourceDirectory, StringComparison.Ordinal))
{
CreateDirectory(
executionContext,
description: "repository source directory",
path: repoPath,
deleteExisting: cleanOption == BuildCleanOption.Source);
}
Trace.Info($"Set repository path for repository {repository.Alias} to '{repoPath}'");
repository.Properties.Set <string>(RepositoryPropertyNames.Path, repoPath);
}
return(newConfig);
}
public async Task RunPluginTaskAsync(IExecutionContext context, string plugin, Dictionary <string, string> inputs, Dictionary <string, string> environment, Variables runtimeVariables, EventHandler <ProcessDataReceivedEventArgs> outputHandler)
{
ArgUtil.NotNullOrEmpty(plugin, nameof(plugin));
// Only allow plugins we defined
if (!_taskPlugins.Contains(plugin))
{
throw new NotSupportedException(plugin);
}
// Resolve the working directory.
string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
ArgUtil.Directory(workingDirectory, nameof(workingDirectory));
// Agent.PluginHost
string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
ArgUtil.File(file, $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
// Agent.PluginHost's arguments
string arguments = $"task \"{plugin}\"";
// construct plugin context
AgentTaskPluginExecutionContext pluginContext = new AgentTaskPluginExecutionContext
{
Inputs = inputs,
Repositories = context.Repositories,
Endpoints = context.Endpoints
};
// variables
foreach (var publicVar in runtimeVariables.Public)
{
pluginContext.Variables[publicVar.Key] = publicVar.Value;
}
foreach (var publicVar in runtimeVariables.Private)
{
pluginContext.Variables[publicVar.Key] = new VariableValue(publicVar.Value, true);
}
// task variables (used by wrapper task)
foreach (var publicVar in context.TaskVariables.Public)
{
pluginContext.TaskVariables[publicVar.Key] = publicVar.Value;
}
foreach (var publicVar in context.TaskVariables.Private)
{
pluginContext.TaskVariables[publicVar.Key] = new VariableValue(publicVar.Value, true);
}
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
var redirectStandardIn = new InputQueue <string>();
redirectStandardIn.Enqueue(JsonUtility.ToString(pluginContext));
processInvoker.OutputDataReceived += outputHandler;
processInvoker.ErrorDataReceived += outputHandler;
// Execute the process. Exit code 0 should always be returned.
// A non-zero exit code indicates infrastructural failure.
// Task failure should be communicated over STDOUT using ## commands.
await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : file,
arguments : arguments,
environment : environment,
requireExitCodeZero : true,
outputEncoding : Encoding.UTF8,
killProcessOnCancel : false,
redirectStandardIn : redirectStandardIn,
cancellationToken : context.CancellationToken);
}
}
public Task StartAsync(IExecutionContext context, List <IStep> steps, CancellationToken token)
{
Trace.Entering();
ArgUtil.NotNull(context, nameof(context));
List <PluginInfo> enabledPlugins = new List <PluginInfo>();
if (context.Variables.GetBoolean("agent.disablelogplugin") ?? false)
{
// all log plugs are disabled
context.Debug("All log plugins are disabled.");
}
else
{
foreach (var plugin in _logPlugins)
{
if (context.Variables.GetBoolean($"agent.disablelogplugin.{plugin.Key}") ?? false)
{
// skip plugin
context.Debug($"Log plugin '{plugin.Key}' is disabled.");
continue;
}
else
{
enabledPlugins.Add(plugin.Value);
}
}
}
if (enabledPlugins.Count > 0)
{
// Resolve the working directory.
string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
ArgUtil.Directory(workingDirectory, nameof(workingDirectory));
// Agent.PluginHost
string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
ArgUtil.File(file, $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
// Agent.PluginHost's arguments
string arguments = $"log \"{_instanceId.ToString("D")}\"";
var processInvoker = HostContext.CreateService <IProcessInvoker>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
if (!string.IsNullOrEmpty(e.Data))
{
_outputs.Enqueue(e.Data);
}
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
if (!string.IsNullOrEmpty(e.Data))
{
_outputs.Enqueue(e.Data);
}
};
_pluginHostProcess = processInvoker.ExecuteAsync(workingDirectory: workingDirectory,
fileName: file,
arguments: arguments,
environment: null,
requireExitCodeZero: true,
outputEncoding: Encoding.UTF8,
killProcessOnCancel: true,
redirectStandardIn: _redirectedStdin,
cancellationToken: token);
// construct plugin context
AgentLogPluginHostContext pluginContext = new AgentLogPluginHostContext
{
PluginAssemblies = new List <string>(),
Repositories = context.Repositories,
Endpoints = context.Endpoints,
Variables = new Dictionary <string, VariableValue>(),
Steps = new Dictionary <string, Pipelines.TaskStepDefinitionReference>()
};
// plugins
pluginContext.PluginAssemblies.AddRange(_logPlugins.Values.Select(x => x.AssemblyName));
// variables
foreach (var publicVar in context.Variables.Public)
{
pluginContext.Variables[publicVar.Key] = publicVar.Value;
}
foreach (var privateVar in context.Variables.Private)
{
pluginContext.Variables[privateVar.Key] = new VariableValue(privateVar.Value, true);
}
// steps
foreach (var step in steps)
{
var taskStep = step as ITaskRunner;
if (taskStep != null)
{
pluginContext.Steps[taskStep.ExecutionContext.Id.ToString("D")] = taskStep.Task.Reference;
}
}
Trace.Info("Send serialized context through STDIN");
_redirectedStdin.Enqueue(JsonUtility.ToString(pluginContext));
foreach (var plugin in _logPlugins)
{
context.Output($"Plugin: '{plugin.Value.FriendlyName}' is running in background.");
}
}
return(Task.CompletedTask);
}
private async Task ProcessPluginCommandAsync(IAsyncCommandContext context, AgentCommandPluginExecutionContext pluginContext, string plugin, Command command, CancellationToken token)
{
// Resolve the working directory.
string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
ArgUtil.Directory(workingDirectory, nameof(workingDirectory));
// Agent.PluginHost
string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
// Agent.PluginHost's arguments
string arguments = $"command \"{plugin}\"";
// Execute the process. Exit code 0 should always be returned.
// A non-zero exit code indicates infrastructural failure.
// Any content coming from STDERR will indicate the command process failed.
// We can't use ## command for plugin to communicate, since we are already processing ## command
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
object stderrLock = new object();
List <string> stderr = new List <string>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
context.Output(e.Data);
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
lock (stderrLock)
{
stderr.Add(e.Data);
};
};
var redirectStandardIn = new InputQueue <string>();
redirectStandardIn.Enqueue(JsonUtility.ToString(pluginContext));
int returnCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : file,
arguments : arguments,
environment : null,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
redirectStandardIn : redirectStandardIn,
cancellationToken : token);
if (returnCode != 0)
{
context.Output(string.Join(Environment.NewLine, stderr));
throw new ProcessExitCodeException(returnCode, file, arguments);
}
else if (stderr.Count > 0)
{
throw new InvalidOperationException(string.Join(Environment.NewLine, stderr));
}
else
{
// Everything works fine.
// Return code is 0.
// No STDERR comes out.
}
}
}
public static HashSet <string> GetUserAttributes(this IRequest request)
{
if (request == null)
{
return(TypeConstants <string> .EmptyHashSet);
}
if (request.Items.TryGetValue(Keywords.Attributes, out var oAttrs))
{
return((HashSet <string>)oAttrs);
}
var authSession = request.GetSession();
var attrs = new HashSet <string>();
if (authSession?.IsAuthenticated == true)
{
attrs.Add("auth");
if (HostContext.HasValidAuthSecret(request))
{
attrs.Add(RoleNames.Admin);
}
if (authSession.Roles != null)
{
foreach (var role in authSession.Roles)
{
attrs.Add("role:" + role);
}
}
if (authSession.Permissions != null)
{
foreach (var perm in authSession.Permissions)
{
attrs.Add("perm:" + perm);
}
}
if (authSession is IAuthSessionExtended extended)
{
if (extended.Scopes != null)
{
foreach (var item in extended.Scopes)
{
attrs.Add("scope:" + item);
}
}
}
var claims = request.GetClaims();
if (claims != null)
{
foreach (var claim in claims)
{
attrs.Add("claim:" + claim);
}
}
}
request.Items[Keywords.Attributes] = attrs;
return(attrs);
}
private Assembly ResolveAssembly(AssemblyLoadContext context, AssemblyName assembly)
{
string assemblyFilename = assembly.Name + ".dll";
return(context.LoadFromAssemblyPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), assemblyFilename)));
}
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = PlanUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Resources.Endpoints;
// Variables
Variables = new Variables(HostContext, message.Variables);
// Environment variables shared across all actions
EnvironmentVariables = new Dictionary <string, string>(VarUtil.EnvironmentVariableKeyComparer);
// Job Outputs
JobOutputs = new Dictionary <string, VariableValue>(StringComparer.OrdinalIgnoreCase);
// Service container info
ServiceContainers = new List <ContainerInfo>();
// Steps context (StepsRunner manages adding the scoped steps context)
StepsContext = new StepsContext();
// Scopes
Scopes = new Dictionary <String, ContextScope>(StringComparer.OrdinalIgnoreCase);
if (message.Scopes?.Count > 0)
{
foreach (var scope in message.Scopes)
{
Scopes[scope.Name] = scope;
}
}
// File table
FileTable = new List <String>(message.FileTable ?? new string[0]);
// Expression functions
if (Variables.GetBoolean("System.HashFilesV2") == true)
{
ExpressionConstants.UpdateFunction <Handlers.HashFiles>("hashFiles", 1, byte.MaxValue);
}
// Expression values
if (message.ContextData?.Count > 0)
{
foreach (var pair in message.ContextData)
{
ExpressionValues[pair.Key] = pair.Value;
}
}
ExpressionValues["secrets"] = Variables.ToSecretsContext();
ExpressionValues["runner"] = new RunnerContext();
ExpressionValues["job"] = new JobContext();
Trace.Info("Initialize GitHub context");
var githubAccessToken = new StringContextData(Variables.Get("system.github.token"));
var base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{githubAccessToken}"));
HostContext.SecretMasker.AddValue(base64EncodedToken);
var githubJob = Variables.Get("system.github.job");
var githubContext = new GitHubContext();
githubContext["token"] = githubAccessToken;
if (!string.IsNullOrEmpty(githubJob))
{
githubContext["job"] = new StringContextData(githubJob);
}
var githubDictionary = ExpressionValues["github"].AssertDictionary("github");
foreach (var pair in githubDictionary)
{
githubContext[pair.Key] = pair.Value;
}
ExpressionValues["github"] = githubContext;
Trace.Info("Initialize Env context");
#if OS_WINDOWS
ExpressionValues["env"] = new DictionaryContextData();
#else
ExpressionValues["env"] = new CaseSensitiveDictionaryContextData();
#endif
// Prepend Path
PrependPath = new List <string>();
// JobSteps for job ExecutionContext
JobSteps = new Queue <IStep>();
// PostJobSteps for job ExecutionContext
PostJobSteps = new Stack <IStep>();
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobDisplayName,
refName: message.JobName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Initialize 'echo on action command success' property, default to false, unless Step_Debug is set
EchoOnActionCommand = Variables.Step_Debug ?? false;
// Verbosity (from GitHub.Step_Debug).
WriteDebug = Variables.Step_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
private async Task EnsureDispatchFinished(WorkerDispatcher jobDispatch, bool cancelRunningJob = false)
{
if (!jobDispatch.WorkerDispatch.IsCompleted)
{
if (cancelRunningJob)
{
// cancel running job when shutting down the agent.
// this will happen when agent get Ctrl+C or message queue loop crashed.
jobDispatch.WorkerCancellationTokenSource.Cancel();
// wait for worker process exit then return.
await jobDispatch.WorkerDispatch;
return;
}
// base on the current design, server will only send one job for a given agent everytime.
// if the agent received a new job request while a previous job request is still running, this typically indicate two situations
// 1. an agent bug cause server and agent mismatch on the state of the job request, ex. agent not renew jobrequest properly but think it still own the job reqest, however server already abandon the jobrequest.
// 2. a server bug or design change that allow server send more than one job request to an given agent that haven't finish previous job request.
var agentServer = HostContext.GetService <IAgentServer>();
TaskAgentJobRequest request = null;
try
{
request = await agentServer.GetAgentRequestAsync(_poolId, jobDispatch.RequestId, CancellationToken.None);
}
catch (Exception ex)
{
// we can't even query for the jobrequest from server, something totally busted, stop agent/worker.
Trace.Error($"Catch exception while checking jobrequest {jobDispatch.JobId} status. Cancel running worker right away.");
Trace.Error(ex);
jobDispatch.WorkerCancellationTokenSource.Cancel();
// make sure worker process exit before we rethrow, otherwise we might leave orphan worker process behind.
await jobDispatch.WorkerDispatch;
// rethrow original exception
throw;
}
if (request.Result != null)
{
// job request has been finished, the server already has result.
// this means agent is busted since it still running that request.
// cancel the zombie worker, run next job request.
Trace.Error($"Received job request while previous job {jobDispatch.JobId} still running on worker. Cancel the previous job since the job request have been finished on server side with result: {request.Result.Value}.");
jobDispatch.WorkerCancellationTokenSource.Cancel();
// wait 45 sec for worker to finish.
Task completedTask = await Task.WhenAny(jobDispatch.WorkerDispatch, Task.Delay(TimeSpan.FromSeconds(45)));
if (completedTask != jobDispatch.WorkerDispatch)
{
// at this point, the job exectuion might encounter some dead lock and even not able to be canclled.
// no need to localize the exception string should never happen.
throw new InvalidOperationException($"Job dispatch process for {jobDispatch.JobId} has encountered unexpected error, the dispatch task is not able to be canceled within 45 seconds.");
}
}
else
{
// something seriously wrong on server side. stop agent from continue running.
// no need to localize the exception string should never happen.
throw new InvalidOperationException($"Server send a new job request while the previous job request {jobDispatch.JobId} haven't finished.");
}
}
try
{
await jobDispatch.WorkerDispatch;
Trace.Info($"Job request {jobDispatch.JobId} processed succeed.");
}
catch (Exception ex)
{
Trace.Error($"Worker Dispatch failed witn an exception for job request {jobDispatch.JobId}.");
Trace.Error(ex);
}
finally
{
WorkerDispatcher workerDispatcher;
if (_jobInfos.TryRemove(jobDispatch.JobId, out workerDispatcher))
{
Trace.Verbose($"Remove WorkerDispather from {nameof(_jobInfos)} dictionary for job {jobDispatch.JobId}.");
workerDispatcher.Dispose();
}
}
}
public CSharpGenerator(MetadataTypesConfig config)
{
Config = config;
feature = HostContext.GetPlugin <NativeTypesFeature>();
}
private async Task RunAsync(AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = message.LockToken;
// start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request with result Cancelled
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
Task <int> workerProcessTask = null;
object _outputLock = new object();
List <string> workerOutput = new List <string>();
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (_outputLock)
{
workerOutput.Add(stdout.Data);
}
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (_outputLock)
{
workerOutput.Add(stderr.Data);
}
}
};
// Start the child process.
var assemblyDirectory = IOUtil.GetBinPath();
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: true,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info($"Send job request message to worker for job {message.JobId}.");
using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 30 sec. kill worker.
Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
// we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner.
await notification.JobStarted(message.JobId);
try
{
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);
string detailInfo = null;
if (!TaskResultUtil.IsValidReturnCode(returnCode))
{
detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandle exception or app crash, attach worker stdout/stderr to JobRequest result.");
}
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request for job {message.JobId} with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result));
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
// print out unhandle exception happened in worker after we complete job request.
// when we run out of disk space, report back to server has higher prority.
if (!string.IsNullOrEmpty(detailInfo))
{
Trace.Error("Unhandle exception happened in worker:");
Trace.Error(detailInfo);
}
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after worker cancel timeout
try
{
Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
{
var messageType = MessageType.CancelRequest;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
messageType = MessageType.AgentShutdown;
break;
case ShutdownReason.OperatingSystemShutdown:
messageType = MessageType.OperatingSystemShutdown;
break;
}
}
await processChannel.SendAsync(
messageType : messageType,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
// wait worker to exit
// if worker doesn't exit within timeout, then kill worker.
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
// worker haven't exit within cancellation timeout.
if (completedTask != workerProcessTask)
{
Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
}
finally
{
// This should be the last thing to run so we don't notify external parties until actually finished
await notification.JobCompleted(message.JobId);
}
}
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory));
// Warn about legacy handler.
if (!ExecutionContext.Variables.Allow_Legacy_Powershell)
{
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error($"Task '{this.Task.Name}' ({this.Task.Version}) is using deprecated task execution handler. The task should use the supported task-lib: https://aka.ms/tasklib");
return;
}
else
{
ExecutionContext.Warning($"Task '{this.Task.Name}' ({this.Task.Version}) is using deprecated task execution handler which will be removed by agent version 2.166.0 (targeted for March 1, 2020). This task must be updated to use the supported task-lib: https://aka.ms/tasklib");
}
// Resolve the target script.
string target = GetTarget();
ArgUtil.NotNullOrEmpty(target, nameof(target));
string scriptFile = Path.Combine(TaskDirectory, target);
ArgUtil.File(scriptFile, nameof(scriptFile));
// Determine the working directory.
string workingDirectory = GetWorkingDirectory();
if (String.IsNullOrEmpty(workingDirectory))
{
workingDirectory = Path.GetDirectoryName(scriptFile);
}
else
{
if (!Directory.Exists(workingDirectory))
{
Directory.CreateDirectory(workingDirectory);
}
}
// Copy the OM binaries into the legacy host folder.
ExecutionContext.Output(StringUtil.Loc("PrepareTaskExecutionHandler"));
IOUtil.CopyDirectory(
source: HostContext.GetDirectory(WellKnownDirectory.ServerOM),
target: HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost),
cancellationToken: ExecutionContext.CancellationToken);
Trace.Info("Finished copying files.");
// Add the legacy ps host environment variables.
AddLegacyHostEnvironmentVariables(scriptFile: scriptFile, workingDirectory: workingDirectory);
AddPrependPathToEnvironment();
// Add proxy setting to LegacyVSTSPowerShellHost.exe.config
var agentProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentProxy.ProxyAddress))
{
AddProxySetting(agentProxy);
}
// Invoke the process.
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += OnDataReceived;
processInvoker.ErrorDataReceived += OnDataReceived;
try
{
String vstsPSHostExe = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), "LegacyVSTSPowerShellHost.exe");
Int32 exitCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : vstsPSHostExe,
arguments : "",
environment : Environment,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
redirectStandardIn : null,
inheritConsoleHandler : !ExecutionContext.Variables.Retain_Default_Encoding,
cancellationToken : ExecutionContext.CancellationToken);
// the exit code from vstsPSHost.exe indicate how many error record we get during execution
// -1 exit code means infrastructure failure of Host itself.
// this is to match current handler's logic.
if (exitCode > 0)
{
if (ExecutionContext.Result != null)
{
ExecutionContext.Debug($"Task result already set. Not failing due to error count ({exitCode}).");
}
else
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("PSScriptError", exitCode));
}
}
else if (exitCode < 0)
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("VSTSHostNonZeroReturn", exitCode));
}
}
finally
{
processInvoker.OutputDataReceived -= OnDataReceived;
processInvoker.ErrorDataReceived -= OnDataReceived;
}
}
}
private async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToken, TaskCompletionSource <int> firstJobRequestRenewed, CancellationToken token)
{
var agentServer = HostContext.GetService <IAgentServer>();
int firstRenewRetryLimit = 5;
TaskAgentJobRequest request = null;
// renew lock during job running.
// stop renew only if cancellation token for lock renew task been signal or exception still happen after retry.
while (!token.IsCancellationRequested)
{
try
{
request = await agentServer.RenewAgentRequestAsync(poolId, requestId, lockToken, token);
Trace.Info($"Successfully renew job request {requestId}, job is valid till {request.LockedUntil.Value}");
if (!firstJobRequestRenewed.Task.IsCompleted)
{
// fire first renew successed event.
firstJobRequestRenewed.TrySetResult(0);
}
// renew again after 60 sec delay
await Task.Delay(TimeSpan.FromSeconds(60), token);
}
catch (TaskAgentJobNotFoundException)
{
// no need for retry. the job is not valid anymore.
Trace.Info($"TaskAgentJobNotFoundException received when renew job request {requestId}, job is no longer valid, stop renew job request.");
return;
}
catch (TaskAgentJobTokenExpiredException)
{
// no need for retry. the job is not valid anymore.
Trace.Info($"TaskAgentJobTokenExpiredException received renew job request {requestId}, job is no longer valid, stop renew job request.");
return;
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
// OperationCanceledException may caused by http timeout or _lockRenewalTokenSource.Cance();
// Stop renew only on cancellation token fired.
Trace.Info($"job renew has been canceled, stop renew job request {requestId}.");
return;
}
catch (Exception ex)
{
Trace.Error($"Catch exception during renew agent jobrequest {requestId}.");
Trace.Error(ex);
// retry
TimeSpan remainingTime = TimeSpan.Zero;
if (!firstJobRequestRenewed.Task.IsCompleted)
{
// retry 5 times every 10 sec for the first renew
if (firstRenewRetryLimit-- > 0)
{
remainingTime = TimeSpan.FromSeconds(10);
}
}
else
{
// retry till reach lockeduntil
remainingTime = request.LockedUntil.Value - DateTime.UtcNow;
}
if (remainingTime > TimeSpan.Zero)
{
Trace.Info($"Retrying lock renewal for jobrequest {requestId}. Job is still locked for: {remainingTime.TotalSeconds} seconds.");
TimeSpan delayTime = remainingTime.TotalSeconds > 60 ? TimeSpan.FromMinutes(1) : remainingTime;
try
{
await Task.Delay(delayTime, token);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info($"job renew has been canceled, stop renew job request {requestId}.");
}
}
else
{
Trace.Info($"Lock renewal has run out of retry, stop renew lock for jobrequest {requestId}.");
return;
}
}
}
}
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_dockerManger = HostContext.GetService <IDockerCommandManager>();
_containerNetwork = $"vsts_network_{Guid.NewGuid().ToString("N")}";
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(Data, nameof(Data));
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory));
// Resolve the target script.
ArgUtil.NotNullOrEmpty(Data.Target, nameof(Data.Target));
string scriptFile = Path.Combine(TaskDirectory, Data.Target);
ArgUtil.File(scriptFile, nameof(scriptFile));
// Determine the working directory.
string workingDirectory = Data.WorkingDirectory;
if (String.IsNullOrEmpty(workingDirectory))
{
workingDirectory = Path.GetDirectoryName(scriptFile);
}
else
{
if (!Directory.Exists(workingDirectory))
{
Directory.CreateDirectory(workingDirectory);
}
}
// scriptName
AddEnvironmentVariable("VSTSPSHOSTSCRIPTNAME", scriptFile);
// workingFolder
AddEnvironmentVariable("VSTSPSHOSTWORKINGFOLDER", workingDirectory);
// outputPreference
AddEnvironmentVariable("VSTSPSHOSTOUTPUTPREFER", ExecutionContext.WriteDebug ? "Continue" : "SilentlyContinue");
// inputParameters
if (Inputs.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTINPUTPARAMETER", JsonUtility.ToString(Inputs));
}
List <String> arguments = new List <string>();
Dictionary <String, String> argumentParameters = new Dictionary <String, String>();
if (string.IsNullOrEmpty(Data.ArgumentFormat))
{
// treatInputsAsArguments
AddEnvironmentVariable("VSTSPSHOSTINPUTISARG", "True");
}
else
{
MatchCollection matches = _argumentMatching.Matches(Data.ArgumentFormat);
if (matches[0].Value.StartsWith("-"))
{
String currentKey = String.Empty;
foreach (Match match in matches)
{
if (match.Value.StartsWith("-"))
{
currentKey = match.Value.Trim('-');
argumentParameters.Add(currentKey, String.Empty);
}
else if (!match.Value.StartsWith("-") && !String.IsNullOrEmpty(currentKey))
{
argumentParameters[currentKey] = match.Value;
currentKey = String.Empty;
}
else
{
throw new Exception($"Found value {match.Value} with no corresponding named parameter");
}
}
}
else
{
foreach (Match match in matches)
{
arguments.Add(match.Value);
}
}
// arguments
if (arguments.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTARGS", JsonUtility.ToString(arguments));
}
// argumentParameters
if (argumentParameters.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTARGPARAMETER", JsonUtility.ToString(argumentParameters));
}
}
// push all variable.
foreach (var variable in ExecutionContext.Variables.Public.Concat(ExecutionContext.Variables.Private))
{
AddEnvironmentVariable("VSTSPSHOSTVAR_" + variable.Key, variable.Value);
}
// push all public variable.
foreach (var variable in ExecutionContext.Variables.Public)
{
AddEnvironmentVariable("VSTSPSHOSTPUBVAR_" + variable.Key, variable.Value);
}
// push all endpoints
List <String> ids = new List <string>();
foreach (ServiceEndpoint endpoint in ExecutionContext.Endpoints)
{
string partialKey = null;
if (string.Equals(endpoint.Name, ServiceEndpoints.SystemVssConnection, StringComparison.OrdinalIgnoreCase))
{
partialKey = ServiceEndpoints.SystemVssConnection.ToUpperInvariant();
AddEnvironmentVariable("VSTSPSHOSTSYSTEMENDPOINT_URL", endpoint.Url.ToString());
AddEnvironmentVariable("VSTSPSHOSTSYSTEMENDPOINT_AUTH", JsonUtility.ToString(endpoint.Authorization));
}
else
{
if (endpoint.Id == Guid.Empty && endpoint.Data.ContainsKey("repositoryId"))
{
partialKey = endpoint.Data["repositoryId"].ToUpperInvariant();
}
else
{
partialKey = endpoint.Id.ToString("D").ToUpperInvariant();
}
ids.Add(partialKey);
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_URL_" + partialKey, endpoint.Url.ToString());
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_NAME_" + partialKey, endpoint.Name);
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_TYPE_" + partialKey, endpoint.Type);
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_AUTH_" + partialKey, JsonUtility.ToString(endpoint.Authorization));
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_DATA_" + partialKey, JsonUtility.ToString(endpoint.Data));
}
}
if (ids.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_IDS", JsonUtility.ToString(ids));
}
// Invoke the process.
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += OnDataReceived;
processInvoker.ErrorDataReceived += OnDataReceived;
try
{
String vstsPSHostExe = Path.Combine(IOUtil.GetExternalsPath(), "vstshost", "LegacyVSTSPowerShellHost.exe");
Int32 exitCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : vstsPSHostExe,
arguments : "",
environment : Environment,
cancellationToken : ExecutionContext.CancellationToken);
// the exit code from vstsPSHost.exe indicate how many error record we get during execution
// -1 exit code means infrastructure failure of Host itself.
// this is to match current handler's logic.
if (exitCode > 0)
{
if (ExecutionContext.Result != null)
{
ExecutionContext.Debug($"Task result already set. Not failing due to error count ({exitCode}).");
}
else
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("PSScriptError", exitCode));
}
}
else if (exitCode < 0)
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("VSTSHostNonZeroReturn", exitCode));
}
}
finally
{
processInvoker.OutputDataReceived -= OnDataReceived;
processInvoker.ErrorDataReceived -= OnDataReceived;
}
}
}
private async Task SetupMessage(HostContext context, Pipelines.AgentJobRequestMessage message)
{
// The agent assumes the server creates this
var jobServer = context.GetService <IJobServer>();
await jobServer.CreateTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, default(CancellationToken));
}
public async Task GetSourceAsync(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
CancellationToken cancellationToken)
{
Trace.Entering();
// Validate args.
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(endpoint, nameof(endpoint));
executionContext.Output($"Syncing repository: {endpoint.Name} ({RepositoryType})");
Uri repositoryUrl = endpoint.Url;
if (!repositoryUrl.IsAbsoluteUri)
{
throw new InvalidOperationException("Repository url need to be an absolute uri.");
}
string targetPath = GetEndpointData(endpoint, Constants.EndpointData.SourcesDirectory);
string sourceBranch = GetEndpointData(endpoint, Constants.EndpointData.SourceBranch);
string sourceVersion = GetEndpointData(endpoint, Constants.EndpointData.SourceVersion);
bool clean = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.Clean))
{
clean = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.Clean]);
}
bool checkoutSubmodules = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.CheckoutSubmodules))
{
checkoutSubmodules = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.CheckoutSubmodules]);
}
bool checkoutNestedSubmodules = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.CheckoutNestedSubmodules))
{
checkoutNestedSubmodules = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.CheckoutNestedSubmodules]);
}
int fetchDepth = 0;
if (endpoint.Data.ContainsKey("fetchDepth") &&
(!int.TryParse(endpoint.Data["fetchDepth"], out fetchDepth) || fetchDepth < 0))
{
fetchDepth = 0;
}
// prefer feature variable over endpoint data
fetchDepth = executionContext.Variables.GetInt(Constants.Variables.Features.GitShallowDepth) ?? fetchDepth;
bool gitLfsSupport = false;
if (endpoint.Data.ContainsKey("GitLfsSupport"))
{
gitLfsSupport = StringUtil.ConvertToBoolean(endpoint.Data["GitLfsSupport"]);
}
// prefer feature variable over endpoint data
gitLfsSupport = executionContext.Variables.GetBoolean(Constants.Variables.Features.GitLfsSupport) ?? gitLfsSupport;
bool exposeCred = executionContext.Variables.GetBoolean(Constants.Variables.System.EnableAccessToken) ?? false;
Trace.Info($"Repository url={repositoryUrl}");
Trace.Info($"targetPath={targetPath}");
Trace.Info($"sourceBranch={sourceBranch}");
Trace.Info($"sourceVersion={sourceVersion}");
Trace.Info($"clean={clean}");
Trace.Info($"checkoutSubmodules={checkoutSubmodules}");
Trace.Info($"checkoutNestedSubmodules={checkoutNestedSubmodules}");
Trace.Info($"exposeCred={exposeCred}");
Trace.Info($"fetchDepth={fetchDepth}");
Trace.Info($"gitLfsSupport={gitLfsSupport}");
// Determine which git will be use
// On windows, we prefer the built-in portable git within the agent's externals folder, set system.prefergitfrompath=true can change the behavior,
// agent will find git.exe from %PATH%
// On Linux, we will always use git find in %PATH% regardless of system.prefergitfrompath
bool preferGitFromPath = executionContext.Variables.GetBoolean(Constants.Variables.System.PreferGitFromPath) ?? false;
#if !OS_WINDOWS
// On linux/OSX, we will always find git from %PATH%
preferGitFromPath = true;
#endif
// Determine do we need to provide creds to git operation
_selfManageGitCreds = executionContext.Variables.GetBoolean(Constants.Variables.System.SelfManageGitCreds) ?? false;
if (_selfManageGitCreds)
{
// Customer choose to own git creds by themselves.
executionContext.Output(StringUtil.Loc("SelfManageGitCreds"));
}
// Initialize git command manager
_gitCommandManager = HostContext.GetService <IGitCommandManager>();
await _gitCommandManager.LoadGitExecutionInfo(executionContext, useBuiltInGit : !preferGitFromPath);
// Make sure the build machine met all requirements for the git repository
// For now, the only requirement we have is git version greater than 2.9 for on-prem tfsgit
RequirementCheck(executionContext, endpoint);
// retrieve credential from endpoint.
string username = string.Empty;
string password = string.Empty;
if (!_selfManageGitCreds && endpoint.Authorization != null)
{
switch (endpoint.Authorization.Scheme)
{
case EndpointAuthorizationSchemes.OAuth:
username = EndpointAuthorizationSchemes.OAuth;
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out password))
{
password = string.Empty;
}
break;
case EndpointAuthorizationSchemes.UsernamePassword:
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Username, out username))
{
// leave the username as empty, the username might in the url, like: http://[email protected]
username = string.Empty;
}
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Password, out password))
{
// we have username, but no password
password = string.Empty;
}
break;
default:
executionContext.Warning($"Unsupport endpoint authorization schemes: {endpoint.Authorization.Scheme}");
break;
}
}
// prepare credentail embedded urls
_repositoryUrlWithCred = UrlUtil.GetCredentialEmbeddedUrl(repositoryUrl, username, password);
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl))
{
_proxyUrlWithCred = UrlUtil.GetCredentialEmbeddedUrl(new Uri(executionContext.Variables.Agent_ProxyUrl), executionContext.Variables.Agent_ProxyUsername, executionContext.Variables.Agent_ProxyPassword);
// uri.absoluteuri will not contains port info if the scheme is http/https and the port is 80/443
// however, git.exe always require you provide port info, if nothing passed in, it will use 1080 as default
// as result, we need prefer the uri.originalstring when it's different than uri.absoluteuri.
if (string.Equals(_proxyUrlWithCred.AbsoluteUri, _proxyUrlWithCred.OriginalString, StringComparison.OrdinalIgnoreCase))
{
_proxyUrlWithCredString = _proxyUrlWithCred.AbsoluteUri;
}
else
{
_proxyUrlWithCredString = _proxyUrlWithCred.OriginalString;
}
}
if (gitLfsSupport)
{
// Construct git-lfs url
UriBuilder gitLfsUrl = new UriBuilder(_repositoryUrlWithCred);
if (gitLfsUrl.Path.EndsWith(".git"))
{
gitLfsUrl.Path = gitLfsUrl.Path + "/info/lfs";
}
else
{
gitLfsUrl.Path = gitLfsUrl.Path + ".git/info/lfs";
}
_gitLfsUrlWithCred = gitLfsUrl.Uri;
}
// Check the current contents of the root folder to see if there is already a repo
// If there is a repo, see if it matches the one we are expecting to be there based on the remote fetch url
// if the repo is not what we expect, remove the folder
if (!await IsRepositoryOriginUrlMatch(executionContext, targetPath, repositoryUrl))
{
// Delete source folder
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
else
{
// delete the index.lock file left by previous canceled build or any operation casue git.exe crash last time.
string lockFile = Path.Combine(targetPath, ".git\\index.lock");
if (File.Exists(lockFile))
{
try
{
File.Delete(lockFile);
}
catch (Exception ex)
{
executionContext.Debug($"Unable to delete the index.lock file: {lockFile}");
executionContext.Debug(ex.ToString());
}
}
// When repo.clean is selected for a git repo, execute git clean -fdx and git reset --hard HEAD on the current repo.
// This will help us save the time to reclone the entire repo.
// If any git commands exit with non-zero return code or any exception happened during git.exe invoke, fall back to delete the repo folder.
if (clean)
{
Boolean softCleanSucceed = true;
// git clean -fdx
int exitCode_clean = await _gitCommandManager.GitClean(executionContext, targetPath);
if (exitCode_clean != 0)
{
executionContext.Debug($"'git clean -fdx' failed with exit code {exitCode_clean}, this normally caused by:\n 1) Path too long\n 2) Permission issue\n 3) File in use\nFor futher investigation, manually run 'git clean -fdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
// git reset --hard HEAD
if (softCleanSucceed)
{
int exitCode_reset = await _gitCommandManager.GitReset(executionContext, targetPath);
if (exitCode_reset != 0)
{
executionContext.Debug($"'git reset --hard HEAD' failed with exit code {exitCode_reset}\nFor futher investigation, manually run 'git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
// git clean -fdx and git reset --hard HEAD for each submodule
if (checkoutSubmodules)
{
if (softCleanSucceed)
{
int exitCode_submoduleclean = await _gitCommandManager.GitSubmoduleClean(executionContext, targetPath);
if (exitCode_submoduleclean != 0)
{
executionContext.Debug($"'git submodule foreach git clean -fdx' failed with exit code {exitCode_submoduleclean}\nFor futher investigation, manually run 'git submodule foreach git clean -fdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
if (softCleanSucceed)
{
int exitCode_submodulereset = await _gitCommandManager.GitSubmoduleReset(executionContext, targetPath);
if (exitCode_submodulereset != 0)
{
executionContext.Debug($"'git submodule foreach git reset --hard HEAD' failed with exit code {exitCode_submodulereset}\nFor futher investigation, manually run 'git submodule foreach git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
}
if (!softCleanSucceed)
{
//fall back
executionContext.Warning("Unable to run \"git clean -fdx\" and \"git reset --hard HEAD\" successfully, delete source folder instead.");
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
}
}
// if the folder is missing, create it
if (!Directory.Exists(targetPath))
{
Directory.CreateDirectory(targetPath);
}
// if the folder contains a .git folder, it means the folder contains a git repo that matches the remote url and in a clean state.
// we will run git fetch to update the repo.
if (!Directory.Exists(Path.Combine(targetPath, ".git")))
{
// init git repository
int exitCode_init = await _gitCommandManager.GitInit(executionContext, targetPath);
if (exitCode_init != 0)
{
throw new InvalidOperationException($"Unable to use git.exe init repository under {targetPath}, 'git init' failed with exit code: {exitCode_init}");
}
int exitCode_addremote = await _gitCommandManager.GitRemoteAdd(executionContext, targetPath, "origin", repositoryUrl.AbsoluteUri);
if (exitCode_addremote != 0)
{
throw new InvalidOperationException($"Unable to use git.exe add remote 'origin', 'git remote add' failed with exit code: {exitCode_addremote}");
}
}
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(0, "Starting fetch...");
// disable git auto gc
int exitCode_disableGC = await _gitCommandManager.GitDisableAutoGC(executionContext, targetPath);
if (exitCode_disableGC != 0)
{
executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performence of fetching.");
}
// always remove any possible left extraheader setting from git config.
if (await _gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader"))
{
executionContext.Debug("Remove any extraheader setting from git config.");
await RemoveGitConfig(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader", string.Empty);
}
// always remove any possible left proxy setting from git config, the proxy setting may contains credential
if (await _gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.proxy"))
{
executionContext.Debug("Remove any proxy setting from git config.");
await RemoveGitConfig(executionContext, targetPath, $"http.proxy", string.Empty);
}
List <string> additionalFetchArgs = new List <string>();
if (!_selfManageGitCreds)
{
// v2.9 git support provide auth header as cmdline arg.
// as long 2.9 git exist, VSTS repo, TFS repo and Github repo will use this to handle auth challenge.
if (UseAuthHeaderCmdlineArg)
{
additionalFetchArgs.Add($"-c http.extraheader=\"AUTHORIZATION: {GenerateAuthHeader(username, password)}\"");
}
else
{
// Otherwise, inject credential into fetch/push url
// inject credential into fetch url
executionContext.Debug("Inject credential into git remote url.");
ArgUtil.NotNull(_repositoryUrlWithCred, nameof(_repositoryUrlWithCred));
// inject credential into fetch url
executionContext.Debug("Inject credential into git remote fetch url.");
int exitCode_seturl = await _gitCommandManager.GitRemoteSetUrl(executionContext, targetPath, "origin", _repositoryUrlWithCred.AbsoluteUri);
if (exitCode_seturl != 0)
{
throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote fetch url, 'git remote set-url' failed with exit code: {exitCode_seturl}");
}
// inject credential into push url
executionContext.Debug("Inject credential into git remote push url.");
exitCode_seturl = await _gitCommandManager.GitRemoteSetPushUrl(executionContext, targetPath, "origin", _repositoryUrlWithCred.AbsoluteUri);
if (exitCode_seturl != 0)
{
throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote push url, 'git remote set-url --push' failed with exit code: {exitCode_seturl}");
}
}
// Prepare proxy config for fetch.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl))
{
executionContext.Debug($"Config proxy server '{executionContext.Variables.Agent_ProxyUrl}' for git fetch.");
ArgUtil.NotNullOrEmpty(_proxyUrlWithCredString, nameof(_proxyUrlWithCredString));
additionalFetchArgs.Add($"-c http.proxy=\"{_proxyUrlWithCredString}\"");
}
// Prepare gitlfs url for fetch and checkout
if (gitLfsSupport)
{
// Initialize git lfs by execute 'git lfs install'
executionContext.Debug("Setup the global Git hooks for Git LFS.");
int exitCode_lfsInstall = await _gitCommandManager.GitLFSInstall(executionContext, targetPath);
if (exitCode_lfsInstall != 0)
{
throw new InvalidOperationException($"Git-lfs installation failed with exit code: {exitCode_lfsInstall}");
}
// Inject credential into lfs fetch/push url
executionContext.Debug("Inject credential into git-lfs remote url.");
ArgUtil.NotNull(_gitLfsUrlWithCred, nameof(_gitLfsUrlWithCred));
// inject credential into fetch url
executionContext.Debug("Inject credential into git-lfs remote fetch url.");
_configModifications["remote.origin.lfsurl"] = _gitLfsUrlWithCred.AbsoluteUri;
int exitCode_configlfsurl = await _gitCommandManager.GitConfig(executionContext, targetPath, "remote.origin.lfsurl", _gitLfsUrlWithCred.AbsoluteUri);
if (exitCode_configlfsurl != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_configlfsurl}");
}
// inject credential into push url
executionContext.Debug("Inject credential into git-lfs remote push url.");
_configModifications["remote.origin.lfspushurl"] = _gitLfsUrlWithCred.AbsoluteUri;
int exitCode_configlfspushurl = await _gitCommandManager.GitConfig(executionContext, targetPath, "remote.origin.lfspushurl", _gitLfsUrlWithCred.AbsoluteUri);
if (exitCode_configlfspushurl != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_configlfspushurl}");
}
}
}
// If this is a build for a pull request, then include
// the pull request reference as an additional ref.
List <string> additionalFetchSpecs = new List <string>();
if (IsPullRequest(sourceBranch))
{
additionalFetchSpecs.Add("+refs/heads/*:refs/remotes/origin/*");
additionalFetchSpecs.Add(StringUtil.Format("+{0}:{1}", sourceBranch, GetRemoteRefName(sourceBranch)));
}
int exitCode_fetch = await _gitCommandManager.GitFetch(executionContext, targetPath, "origin", fetchDepth, additionalFetchSpecs, string.Join(" ", additionalFetchArgs), cancellationToken);
if (exitCode_fetch != 0)
{
throw new InvalidOperationException($"Git fetch failed with exit code: {exitCode_fetch}");
}
// Checkout
// sourceToBuild is used for checkout
// if sourceBranch is a PR branch or sourceVersion is null, make sure branch name is a remote branch. we need checkout to detached head.
// (change refs/heads to refs/remotes/origin, refs/pull to refs/remotes/pull, or leava it as it when the branch name doesn't contain refs/...)
// if sourceVersion provide, just use that for checkout, since when you checkout a commit, it will end up in detached head.
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(80, "Starting checkout...");
string sourcesToBuild;
if (IsPullRequest(sourceBranch) || string.IsNullOrEmpty(sourceVersion))
{
sourcesToBuild = GetRemoteRefName(sourceBranch);
}
else
{
sourcesToBuild = sourceVersion;
}
// fetch lfs object upfront, this will avoid fetch lfs object during checkout which cause checkout taking forever
// since checkout will fetch lfs object 1 at a time, while git lfs fetch will fetch lfs object in parallel.
if (gitLfsSupport)
{
int exitCode_lfsFetch = await _gitCommandManager.GitLFSFetch(executionContext, targetPath, "origin", sourcesToBuild, string.Join(" ", additionalFetchArgs), cancellationToken);
if (exitCode_lfsFetch != 0)
{
throw new InvalidOperationException($"Git fetch failed with exit code: {exitCode_lfsFetch}");
}
}
// Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw)
int exitCode_checkout = await _gitCommandManager.GitCheckout(executionContext, targetPath, sourcesToBuild, cancellationToken);
if (exitCode_checkout != 0)
{
// local repository is shallow repository, checkout may fail due to lack of commits history.
// this will happen when the checkout commit is older than tip -> fetchDepth
if (fetchDepth > 0)
{
executionContext.Warning(StringUtil.Loc("ShallowCheckoutFail", fetchDepth, sourcesToBuild));
}
throw new InvalidOperationException($"Git checkout failed with exit code: {exitCode_checkout}");
}
// Submodule update
if (checkoutSubmodules)
{
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(90, "Updating submodules...");
List <string> additionalSubmoduleUpdateArgs = new List <string>();
if (!_selfManageGitCreds)
{
if (UseAuthHeaderCmdlineArg)
{
string authorityUrl = repositoryUrl.AbsoluteUri.Replace(repositoryUrl.PathAndQuery, string.Empty);
additionalSubmoduleUpdateArgs.Add($"-c http.{authorityUrl}.extraheader=\"AUTHORIZATION: {GenerateAuthHeader(username, password)}\"");
}
// Prepare proxy config for submodule update.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl))
{
executionContext.Debug($"Config proxy server '{executionContext.Variables.Agent_ProxyUrl}' for git submodule update.");
ArgUtil.NotNullOrEmpty(_proxyUrlWithCredString, nameof(_proxyUrlWithCredString));
additionalSubmoduleUpdateArgs.Add($"-c http.proxy=\"{_proxyUrlWithCredString}\"");
}
}
int exitCode_submoduleUpdate = await _gitCommandManager.GitSubmoduleUpdate(executionContext, targetPath, string.Join(" ", additionalSubmoduleUpdateArgs), checkoutNestedSubmodules, cancellationToken);
if (exitCode_submoduleUpdate != 0)
{
throw new InvalidOperationException($"Git submodule update failed with exit code: {exitCode_submoduleUpdate}");
}
}
// handle expose creds, related to 'Allow Scripts to Access OAuth Token' option
if (!_selfManageGitCreds)
{
if (UseAuthHeaderCmdlineArg && exposeCred)
{
string configKey = $"http.{repositoryUrl.AbsoluteUri}.extraheader";
string configValue = $"\"AUTHORIZATION: {GenerateAuthHeader(username, password)}\"";
_configModifications[configKey] = configValue.Trim('\"');
int exitCode_config = await _gitCommandManager.GitConfig(executionContext, targetPath, configKey, configValue);
if (exitCode_config != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_config}");
}
}
if (!UseAuthHeaderCmdlineArg && !exposeCred)
{
// remove cached credential from origin's fetch/push url.
await RemoveCachedCredential(executionContext, targetPath, repositoryUrl, "origin");
}
if (exposeCred)
{
// save proxy setting to git config.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl))
{
executionContext.Debug($"Save proxy config for proxy server '{executionContext.Variables.Agent_ProxyUrl}' into git config.");
ArgUtil.NotNullOrEmpty(_proxyUrlWithCredString, nameof(_proxyUrlWithCredString));
string proxyConfigKey = "http.proxy";
string proxyConfigValue = $"\"{_proxyUrlWithCredString}\"";
_configModifications[proxyConfigKey] = proxyConfigValue.Trim('\"');
int exitCode_proxyconfig = await _gitCommandManager.GitConfig(executionContext, targetPath, proxyConfigKey, proxyConfigValue);
if (exitCode_proxyconfig != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_proxyconfig}");
}
}
}
if (gitLfsSupport && !exposeCred)
{
executionContext.Debug("Remove git-lfs fetch and push url setting from git config.");
await RemoveGitConfig(executionContext, targetPath, "remote.origin.lfsurl", _gitLfsUrlWithCred.AbsoluteUri);
_configModifications.Remove("remote.origin.lfsurl");
await RemoveGitConfig(executionContext, targetPath, "remote.origin.lfspushurl", _gitLfsUrlWithCred.AbsoluteUri);
_configModifications.Remove("remote.origin.lfspushurl");
}
}
}
