private bool Match(ServerBuildArtifact buildArtifact, ArtifactDefinition artifactDefinition)
{
//TODO: If editing artifactDefinitionName is not allowed then we can remove this
if (string.Equals(artifactDefinition.Name, AllArtifacts, StringComparison.OrdinalIgnoreCase))
{
return(true);
}
if (string.Equals(artifactDefinition.Name, buildArtifact.Name, StringComparison.OrdinalIgnoreCase))
{
return(true);
}
return(false);
}
private static void OutputArtifactDefinition(ArtifactDefinition artifactArtifactDefinition)
{
Log.Warn(" -Definition: ");
Log.Info(" -Description: "+ artifactArtifactDefinition.BusinessDescription);
Log.Info("");
Log.Info(" -Example: "+ artifactArtifactDefinition.BusinessExample);
Log.Info("");
Log.Warn(" -Analogies: ");
foreach (var a in artifactArtifactDefinition.Analogies)
{
Log.Info(" -Analogy: "+ a.Name);
Log.Info(" -Description: "+ a.Description);
}
Log.Info(" -Comments: "+ artifactArtifactDefinition.Comments);
}
public async Task DownloadAsync(
IExecutionContext executionContext,
ArtifactDefinition artifactDefinition,
string localFolderPath)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath));
var gitHubDetails = artifactDefinition.Details as GitHubArtifactDetails;
ArgUtil.NotNull(gitHubDetails, nameof(gitHubDetails));
executionContext.Output(StringUtil.Loc("RMReceivedGithubArtifactDetails"));
ServiceEndpoint endpoint = executionContext.Endpoints.FirstOrDefault((e => string.Equals(e.Name, gitHubDetails.ConnectionName, StringComparison.OrdinalIgnoreCase)));
if (endpoint == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMGitHubEndpointNotFound", gitHubDetails.ConnectionName));
}
ServiceEndpoint gitHubEndpoint = PrepareGitHubTaskEndpoint(endpoint, gitHubDetails.CloneUrl);
var extensionManager = HostContext.GetService <IExtensionManager>();
ISourceProvider sourceProvider = (extensionManager.GetExtensions <ISourceProvider>()).FirstOrDefault(x => x.RepositoryType == Microsoft.TeamFoundation.DistributedTask.Pipelines.RepositoryTypes.GitHub);
if (sourceProvider == null)
{
throw new InvalidOperationException(StringUtil.Loc("SourceArtifactProviderNotFound", Microsoft.TeamFoundation.DistributedTask.Pipelines.RepositoryTypes.GitHub));
}
gitHubEndpoint.Data.Add(Constants.EndpointData.SourcesDirectory, localFolderPath);
gitHubEndpoint.Data.Add(Constants.EndpointData.SourceBranch, gitHubDetails.Branch);
gitHubEndpoint.Data.Add(Constants.EndpointData.SourceVersion, artifactDefinition.Version);
gitHubEndpoint.Data.Add(EndpointData.CheckoutSubmodules, gitHubDetails.CheckoutSubmodules);
gitHubEndpoint.Data.Add(EndpointData.CheckoutNestedSubmodules, gitHubDetails.CheckoutNestedSubmodules);
gitHubEndpoint.Data.Add("fetchDepth", gitHubDetails.FetchDepth);
gitHubEndpoint.Data.Add("GitLfsSupport", gitHubDetails.GitLfsSupport);
try
{
await sourceProvider.GetSourceAsync(executionContext, gitHubEndpoint, executionContext.CancellationToken);
}
catch (InvalidOperationException ex)
{
throw new ArtifactDownloadException(StringUtil.Loc("RMDownloadArtifactUnexpectedError"), ex);
}
}
private ArtifactDefinition ConvertToArtifactDefinition(AgentArtifactDefinition agentArtifactDefinition, IExecutionContext executionContext, IArtifactExtension extension)
{
Trace.Entering();
ArgUtil.NotNull(agentArtifactDefinition, nameof(agentArtifactDefinition));
ArgUtil.NotNull(executionContext, nameof(executionContext));
var artifactDefinition = new ArtifactDefinition
{
ArtifactType = agentArtifactDefinition.ArtifactType,
Name = agentArtifactDefinition.Name,
Version = agentArtifactDefinition.Version
};
artifactDefinition.Details = extension.GetArtifactDetails(executionContext, agentArtifactDefinition);
return(artifactDefinition);
}
public async Task DownloadAsync(IExecutionContext executionContext, ArtifactDefinition artifactDefinition, string downloadFolderPath)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
ArgUtil.NotNullOrEmpty(downloadFolderPath, nameof(downloadFolderPath));
var gitArtifactDetails = artifactDefinition.Details as TfsGitArtifactDetails;
ArgUtil.NotNull(gitArtifactDetails, nameof(gitArtifactDetails));
ServiceEndpoint endpoint = executionContext.Endpoints.FirstOrDefault((e => string.Equals(e.Name, gitArtifactDetails.RepositoryId, StringComparison.OrdinalIgnoreCase)));
if (endpoint == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMGitEndpointNotFound"));
}
var extensionManager = HostContext.GetService <IExtensionManager>();
ISourceProvider sourceProvider = (extensionManager.GetExtensions <ISourceProvider>()).FirstOrDefault(x => x.RepositoryType == RepositoryTypes.TfsGit);
if (sourceProvider == null)
{
throw new InvalidOperationException(StringUtil.Loc("SourceArtifactProviderNotFound", RepositoryTypes.TfsGit));
}
var tfsGitEndpoint = endpoint.Clone();
tfsGitEndpoint.Data.Add(Constants.EndpointData.SourcesDirectory, downloadFolderPath);
tfsGitEndpoint.Data.Add(Constants.EndpointData.SourceBranch, gitArtifactDetails.Branch);
tfsGitEndpoint.Data.Add(Constants.EndpointData.SourceVersion, artifactDefinition.Version);
tfsGitEndpoint.Data.Add(EndpointData.CheckoutSubmodules, gitArtifactDetails.CheckoutSubmodules);
tfsGitEndpoint.Data.Add(EndpointData.CheckoutNestedSubmodules, gitArtifactDetails.CheckoutNestedSubmodules);
tfsGitEndpoint.Data.Add("fetchDepth", gitArtifactDetails.FetchDepth);
tfsGitEndpoint.Data.Add("GitLfsSupport", gitArtifactDetails.GitLfsSupport);
try
{
await sourceProvider.GetSourceAsync(executionContext, tfsGitEndpoint, executionContext.CancellationToken);
}
catch (InvalidOperationException ex)
{
throw new ArtifactDownloadException(StringUtil.Loc("RMDownloadArtifactUnexpectedError"), ex);
}
}
private ArtifactDefinition ConvertToArtifactDefinition(AgentArtifactDefinition agentArtifactDefinition, IExecutionContext executionContext, IArtifactExtension extension)
{
Trace.Entering();
ArgUtil.NotNull(agentArtifactDefinition, nameof(agentArtifactDefinition));
ArgUtil.NotNull(executionContext, nameof(executionContext));
var artifactDefinition = new ArtifactDefinition
{
ArtifactType = agentArtifactDefinition.ArtifactType,
Name = agentArtifactDefinition.Name,
Version = agentArtifactDefinition.Version
};
RetryExecutor retryExecutor = new RetryExecutor();
retryExecutor.ShouldRetryAction = (ex) =>
{
bool retry = true;
if (ex is InvalidOperationException)
{
retry = false;
}
else
{
Trace.Warning(ex.ToString());
}
return(retry);
};
retryExecutor.Execute(
() =>
{
artifactDefinition.Details = extension.GetArtifactDetails(executionContext, agentArtifactDefinition);
});
return(artifactDefinition);
}
private TestHostContext Setup([CallerMemberName] string name = "")
{
TestHostContext hc = new TestHostContext(this, name);
_ec = new Mock <IExecutionContext>();
_httpClient = new Mock <IGenericHttpClient>();
_artifactDefinition = new ArtifactDefinition
{
Details = new JenkinsArtifactDetails
{
Url = new Uri("http://localhost"),
JobName = "jenkins",
Alias = "jenkins"
}
};
_extensionManager = new Mock <IExtensionManager>();
hc.SetSingleton <IExtensionManager>(_extensionManager.Object);
hc.SetSingleton <IGenericHttpClient>(_httpClient.Object);
return(hc);
}
public async Task DownloadAsync(IExecutionContext executionContext, ArtifactDefinition artifactDefinition, string downloadFolderPath)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
ArgUtil.NotNullOrEmpty(downloadFolderPath, nameof(downloadFolderPath));
var tfsVcArtifactDetails = artifactDefinition.Details as TfsVCArtifactDetails;
ArgUtil.NotNull(tfsVcArtifactDetails, nameof(tfsVcArtifactDetails));
ServiceEndpoint endpoint = executionContext.Endpoints.FirstOrDefault((e => string.Equals(e.Name, tfsVcArtifactDetails.RepositoryId, StringComparison.OrdinalIgnoreCase)));
if (endpoint == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMTfsVCEndpointNotFound"));
}
var tfsVCEndpoint = endpoint.Clone();
PrepareTfsVCEndpoint(tfsVCEndpoint, tfsVcArtifactDetails);
var extensionManager = HostContext.GetService <IExtensionManager>();
ISourceProvider sourceProvider = (extensionManager.GetExtensions <ISourceProvider>()).FirstOrDefault(x => x.RepositoryType == Microsoft.TeamFoundation.DistributedTask.Pipelines.RepositoryTypes.Tfvc);
if (sourceProvider == null)
{
throw new InvalidOperationException(StringUtil.Loc("SourceArtifactProviderNotFound", Microsoft.TeamFoundation.DistributedTask.Pipelines.RepositoryTypes.Tfvc));
}
var rootDirectory = Directory.GetParent(downloadFolderPath).Name;
executionContext.Variables.Set(Constants.Variables.Agent.BuildDirectory, rootDirectory);
tfsVCEndpoint.Data.Add(Constants.EndpointData.SourcesDirectory, downloadFolderPath);
tfsVCEndpoint.Data.Add(Constants.EndpointData.SourceVersion, artifactDefinition.Version);
await sourceProvider.GetSourceAsync(executionContext, tfsVCEndpoint, executionContext.CancellationToken);
}
public async Task DownloadAsync(
IExecutionContext executionContext,
ArtifactDefinition artifactDefinition,
string localFolderPath)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath));
var gitHubDetails = artifactDefinition.Details as GitHubArtifactDetails;
ArgUtil.NotNull(gitHubDetails, nameof(gitHubDetails));
executionContext.Output(StringUtil.Loc("RMReceivedGithubArtifactDetails"));
ServiceEndpoint endpoint = executionContext.Endpoints.FirstOrDefault((e => string.Equals(e.Name, gitHubDetails.ConnectionName, StringComparison.OrdinalIgnoreCase)));
if (endpoint == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMGitHubEndpointNotFound", gitHubDetails.ConnectionName));
}
ServiceEndpoint gitHubEndpoint = PrepareGitHubTaskEndpoint(endpoint, gitHubDetails.CloneUrl);
var extensionManager = HostContext.GetService <IExtensionManager>();
ISourceProvider sourceProvider = (extensionManager.GetExtensions <ISourceProvider>()).FirstOrDefault(x => x.RepositoryType == WellKnownRepositoryTypes.GitHub);
if (sourceProvider == null)
{
throw new InvalidOperationException(StringUtil.Loc("SourceArtifactProviderNotFound", WellKnownRepositoryTypes.GitHub));
}
gitHubEndpoint.Data.Add(Constants.EndpointData.SourcesDirectory, localFolderPath);
gitHubEndpoint.Data.Add(Constants.EndpointData.SourceBranch, gitHubDetails.Branch);
gitHubEndpoint.Data.Add(Constants.EndpointData.SourceVersion, artifactDefinition.Version);
await sourceProvider.GetSourceAsync(executionContext, gitHubEndpoint, executionContext.CancellationToken);
}
private async Task DownloadArtifactAsync(
IExecutionContext executionContext,
ServerBuildArtifact buildArtifact,
ArtifactDefinition artifactDefinition,
string localFolderPath)
{
var downloadFolderPath = Path.Combine(localFolderPath, buildArtifact.Name);
var buildArtifactDetails = artifactDefinition.Details as BuildArtifactDetails;
if ((buildArtifact.Resource.Type == null && buildArtifact.Id == 0) || // bug on build API Bug 378900
string.Equals(buildArtifact.Resource.Type, ArtifactResourceTypes.FilePath, StringComparison.OrdinalIgnoreCase))
{
executionContext.Output(StringUtil.Loc("RMArtifactTypeFileShare"));
if (!PlatformUtil.RunningOnWindows)
{
throw new NotSupportedException(StringUtil.Loc("RMFileShareArtifactErrorOnNonWindowsAgent"));
}
string fileShare;
if (buildArtifact.Id == 0)
{
fileShare = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath;
}
else
{
fileShare = new Uri(Path.Combine(buildArtifact.Resource.DownloadUrl, buildArtifact.Name)).LocalPath;
if (!Directory.Exists(fileShare))
{
// download path does not exist, log and fall back
var parenthPath = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath;
executionContext.Output(StringUtil.Loc("RMArtifactNameDirectoryNotFound", fileShare, parenthPath));
fileShare = parenthPath;
}
}
if (!Directory.Exists(fileShare))
{
// download path does not exist, raise exception
throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactDirectoryNotFoundError", fileShare, WindowsIdentity.GetCurrent().Name));
}
executionContext.Output(StringUtil.Loc("RMDownloadingArtifactFromFileShare", fileShare, downloadFolderPath));
var fileShareArtifact = new FileShareArtifact();
await fileShareArtifact.DownloadArtifactAsync(executionContext, HostContext, artifactDefinition, fileShare, downloadFolderPath);
}
else if (buildArtifactDetails != null &&
string.Equals(buildArtifact.Resource.Type, ArtifactResourceTypes.Container, StringComparison.OrdinalIgnoreCase))
{
executionContext.Output(StringUtil.Loc("RMArtifactTypeServerDrop"));
// Get containerId and rootLocation for the artifact #/922702/drop
string containerUrl = buildArtifact.Resource.Data;
string[] parts = containerUrl.Split(new[] { '/' }, 3);
if (parts.Length < 3)
{
throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactContainerDetailsNotFoundError", buildArtifact.Name));
}
int containerId;
string rootLocation = parts[2];
if (!int.TryParse(parts[1], out containerId))
{
throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactContainerDetailsInvalidError", buildArtifact.Name));
}
string rootDestinationDir = Path.Combine(localFolderPath, rootLocation);
executionContext.Output(StringUtil.Loc("RMDownloadingArtifactFromFileContainer", containerUrl, rootDestinationDir));
var containerFetchEngineOptions = new ContainerFetchEngineOptions
{
ParallelDownloadLimit = executionContext.Variables.Release_Parallel_Download_Limit ?? ContainerFetchEngineDefaultOptions.ParallelDownloadLimit,
DownloadBufferSize = executionContext.Variables.Release_Download_BufferSize ?? ContainerFetchEngineDefaultOptions.DownloadBufferSize
};
executionContext.Output(StringUtil.Loc("RMParallelDownloadLimit", containerFetchEngineOptions.ParallelDownloadLimit));
executionContext.Output(StringUtil.Loc("RMDownloadBufferSize", containerFetchEngineOptions.DownloadBufferSize));
IContainerProvider containerProvider =
new ContainerProviderFactory(buildArtifactDetails, rootLocation, containerId, executionContext).GetContainerProvider(
ArtifactResourceTypes.Container);
using (var engine = new ContainerFetchEngine.ContainerFetchEngine(containerProvider, rootLocation, rootDestinationDir))
{
engine.ContainerFetchEngineOptions = containerFetchEngineOptions;
engine.ExecutionLogger = new ExecutionLogger(executionContext);
await engine.FetchAsync(executionContext.CancellationToken);
}
}
else
{
executionContext.Warning(StringUtil.Loc("RMArtifactTypeNotSupported", buildArtifact.Resource.Type));
}
}
private async Task DownloadArtifacts(IExecutionContext executionContext, List <AgentArtifactDefinition> agentArtifactDefinitions, string artifactsWorkingFolder)
{
Trace.Entering();
foreach (AgentArtifactDefinition agentArtifactDefinition in agentArtifactDefinitions)
{
// We don't need to check if its old style artifact anymore. All the build data has been fixed and all the build artifact has Alias now.
ArgUtil.NotNullOrEmpty(agentArtifactDefinition.Alias, nameof(agentArtifactDefinition.Alias));
var extensionManager = HostContext.GetService <IExtensionManager>();
IArtifactExtension extension = (extensionManager.GetExtensions <IArtifactExtension>()).FirstOrDefault(x => agentArtifactDefinition.ArtifactType == x.ArtifactType);
if (extension == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMArtifactTypeNotSupported"));
}
Trace.Info($"Found artifact extension of type {extension.ArtifactType}");
executionContext.Output(StringUtil.Loc("RMStartArtifactsDownload"));
ArtifactDefinition artifactDefinition = ConvertToArtifactDefinition(agentArtifactDefinition, executionContext, extension);
executionContext.Output(StringUtil.Loc("RMArtifactDownloadBegin", agentArtifactDefinition.Alias));
executionContext.Output(StringUtil.Loc("RMDownloadArtifactType", agentArtifactDefinition.ArtifactType));
// Get the local path where this artifact should be downloaded.
string downloadFolderPath = Path.GetFullPath(Path.Combine(artifactsWorkingFolder, agentArtifactDefinition.Alias ?? string.Empty));
// Create the directory if it does not exist.
if (!Directory.Exists(downloadFolderPath))
{
// TODO: old windows agent has a directory cache, verify and implement it if its required.
Directory.CreateDirectory(downloadFolderPath);
executionContext.Output(StringUtil.Loc("RMArtifactFolderCreated", downloadFolderPath));
}
// download the artifact to this path.
RetryExecutor retryExecutor = new RetryExecutor();
retryExecutor.ShouldRetryAction = (ex) =>
{
executionContext.Output(StringUtil.Loc("RMErrorDuringArtifactDownload", ex));
bool retry = true;
if (ex is ArtifactDownloadException)
{
retry = false;
}
else
{
executionContext.Output(StringUtil.Loc("RMRetryingArtifactDownload"));
}
return(retry);
};
await retryExecutor.ExecuteAsync(
async() =>
{
//TODO:SetAttributesToNormal
var releaseFileSystemManager = HostContext.GetService <IReleaseFileSystemManager>();
releaseFileSystemManager.CleanupDirectory(downloadFolderPath, executionContext.CancellationToken);
if (agentArtifactDefinition.ArtifactType == AgentArtifactType.GitHub ||
agentArtifactDefinition.ArtifactType == AgentArtifactType.TFGit ||
agentArtifactDefinition.ArtifactType == AgentArtifactType.Tfvc)
{
throw new NotImplementedException();
}
else
{
await extension.DownloadAsync(executionContext, artifactDefinition, downloadFolderPath);
}
});
executionContext.Output(StringUtil.Loc("RMArtifactDownloadFinished", agentArtifactDefinition.Alias));
}
executionContext.Output(StringUtil.Loc("RMArtifactsDownloadFinished"));
}
private async Task DownloadArtifactAsync(
IExecutionContext executionContext,
ServerBuildArtifact buildArtifact,
ArtifactDefinition artifactDefinition,
string localFolderPath,
BuildHttpClient buildClient,
XamlBuildHttpClient xamlBuildClient,
DefinitionType definitionType,
int buildId)
{
var downloadFolderPath = Path.Combine(localFolderPath, buildArtifact.Name);
var buildArtifactDetails = artifactDefinition.Details as BuildArtifactDetails;
if ((buildArtifact.Resource.Type == null && buildArtifact.Id == 0) || // bug on build API Bug 378900
string.Equals(buildArtifact.Resource.Type, WellKnownArtifactResourceTypes.FilePath, StringComparison.OrdinalIgnoreCase))
{
executionContext.Output("Artifact Type: FileShare");
string fileShare;
if (buildArtifact.Id == 0)
{
fileShare = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath;
}
else
{
fileShare = new Uri(Path.Combine(buildArtifact.Resource.DownloadUrl, buildArtifact.Name)).LocalPath;
if (!Directory.Exists(fileShare))
{
// download path does not exist, log and fall back
var parenthPath = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath;
executionContext.Output(StringUtil.Loc("RMArtifactNameDirectoryNotFound", fileShare, parenthPath));
fileShare = parenthPath;
}
}
if (!Directory.Exists(fileShare))
{
// download path does not exist, raise exception
throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactDirectoryNotFoundError", fileShare));
}
executionContext.Output(StringUtil.Loc("RMDownloadingArtifactFromFileShare", fileShare));
var fileShareArtifact = new FileShareArtifact();
await fileShareArtifact.DownloadArtifactAsync(executionContext, HostContext, artifactDefinition, fileShare, downloadFolderPath);
}
else if (buildArtifactDetails != null &&
string.Equals(buildArtifact.Resource.Type, WellKnownArtifactResourceTypes.Container, StringComparison.OrdinalIgnoreCase))
{
executionContext.Output("Artifact Type: ServerDrop");
// Get containerId and rootLocation for the artifact #/922702/drop
string[] parts = buildArtifact.Resource.Data.Split(new[] { '/' }, 3);
if (parts.Length < 3)
{
throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactContainerDetailsNotFoundError", buildArtifact.Name));
}
int containerId;
string rootLocation = parts[2];
if (!int.TryParse(parts[1], out containerId))
{
throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactContainerDetailsInvaidError", buildArtifact.Name));
}
IContainerProvider containerProvider =
new ContainerProviderFactory(buildArtifactDetails, rootLocation, containerId, executionContext).GetContainerProvider(
WellKnownArtifactResourceTypes.Container);
string rootDestinationDir = Path.Combine(localFolderPath, rootLocation);
var containerFetchEngineOptions = new ContainerFetchEngineOptions
{
ParallelDownloadLimit = 4,
CancellationToken = executionContext.CancellationToken
};
using (var engine = new ContainerFetchEngine.ContainerFetchEngine(containerProvider, rootLocation, rootDestinationDir))
{
engine.ContainerFetchEngineOptions = containerFetchEngineOptions;
engine.ExecutionLogger = new ExecutionLogger(executionContext);
await engine.FetchAsync();
}
}
else
{
executionContext.Warning(StringUtil.Loc("RMArtifactTypeNotSupported", buildArtifact.Resource.Type));
}
}
private async Task DownloadArtifacts(IExecutionContext executionContext,
IList <AgentArtifactDefinition> agentArtifactDefinitions, string artifactsWorkingFolder)
{
Trace.Entering();
CreateArtifactsFolder(executionContext, artifactsWorkingFolder);
executionContext.Output(StringUtil.Loc("RMDownloadingArtifact"));
foreach (AgentArtifactDefinition agentArtifactDefinition in agentArtifactDefinitions)
{
// We don't need to check if its old style artifact anymore. All the build data has been fixed and all the build artifact has Alias now.
ArgUtil.NotNullOrEmpty(agentArtifactDefinition.Alias, nameof(agentArtifactDefinition.Alias));
var extensionManager = HostContext.GetService <IExtensionManager>();
IArtifactExtension extension =
(extensionManager.GetExtensions <IArtifactExtension>()).FirstOrDefault(x =>
agentArtifactDefinition.ArtifactType == x.ArtifactType);
if (extension == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMArtifactTypeNotSupported",
agentArtifactDefinition.ArtifactType));
}
Trace.Info($"Found artifact extension of type {extension.ArtifactType}");
executionContext.Output(StringUtil.Loc("RMStartArtifactsDownload"));
ArtifactDefinition artifactDefinition =
ConvertToArtifactDefinition(agentArtifactDefinition, executionContext, extension);
executionContext.Output(StringUtil.Loc("RMArtifactDownloadBegin", agentArtifactDefinition.Alias,
agentArtifactDefinition.ArtifactType));
// Get the local path where this artifact should be downloaded.
string downloadFolderPath = Path.GetFullPath(Path.Combine(artifactsWorkingFolder,
agentArtifactDefinition.Alias ?? string.Empty));
// download the artifact to this path.
RetryExecutor retryExecutor = new RetryExecutor();
retryExecutor.ShouldRetryAction = (ex) =>
{
executionContext.Output(StringUtil.Loc("RMErrorDuringArtifactDownload", ex));
bool retry = true;
if (ex is ArtifactDownloadException)
{
retry = false;
}
else
{
executionContext.Output(StringUtil.Loc("RMRetryingArtifactDownload"));
Trace.Warning(ex.ToString());
}
return(retry);
};
await retryExecutor.ExecuteAsync(
async() =>
{
var releaseFileSystemManager = HostContext.GetService <IReleaseFileSystemManager>();
executionContext.Output(StringUtil.Loc("RMEnsureArtifactFolderExistsAndIsClean",
downloadFolderPath));
releaseFileSystemManager.EnsureEmptyDirectory(downloadFolderPath,
executionContext.CancellationToken);
await extension.DownloadAsync(executionContext, artifactDefinition, downloadFolderPath);
});
executionContext.Output(StringUtil.Loc("RMArtifactDownloadFinished",
agentArtifactDefinition.Alias));
}
executionContext.Output(StringUtil.Loc("RMArtifactsDownloadFinished"));
}
public async Task DownloadCommitsAsync(IExecutionContext context, ArtifactDefinition artifactDefinition, string commitsWorkFolder)
{
ArgUtil.NotNull(context, nameof(context));
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
Trace.Entering();
var jenkinsDetails = artifactDefinition.Details as JenkinsArtifactDetails;
int startJobId = 0, endJobId = 0;
if (!string.IsNullOrEmpty(jenkinsDetails.EndCommitArtifactVersion))
{
if (int.TryParse(jenkinsDetails.EndCommitArtifactVersion, out endJobId))
{
context.Output(StringUtil.Loc("RMDownloadingCommits"));
if (int.TryParse(jenkinsDetails.StartCommitArtifactVersion, out startJobId))
{
if (startJobId < endJobId)
{
context.Output(StringUtil.Loc("DownloadingJenkinsCommitsBetween", startJobId, endJobId));
}
else if (startJobId > endJobId)
{
context.Output(StringUtil.Loc("JenkinsRollbackDeployment", startJobId, endJobId));
// swap the job IDs to fetch the roll back commits
int swap = startJobId;
startJobId = endJobId;
endJobId = swap;
}
else if (startJobId == endJobId)
{
context.Output(StringUtil.Loc("JenkinsNoCommitsToFetch"));
return;
}
}
else
{
context.Debug(StringUtil.Loc("JenkinsDownloadingChangeFromCurrentBuild"));
}
try
{
IEnumerable <Change> changes = await DownloadCommits(context, jenkinsDetails, startJobId, endJobId);
if (changes.Any())
{
string commitsFileName = GetCommitsFileName(jenkinsDetails.Alias);
string commitsFilePath = Path.Combine(commitsWorkFolder, commitsFileName);
context.Debug($"Commits will be written to {commitsFilePath}");
WriteCommitsToFile(context, changes, commitsFilePath);
context.Debug($"Commits written to {commitsFilePath}");
context.QueueAttachFile(CoreAttachmentType.FileAttachment, commitsFileName, commitsFilePath);
}
}
catch (SocketException ex)
{
context.AddIssue(new Issue {
Type = IssueType.Error, Message = $"SocketException occurred. { ex.Message }." +
$"Verify whether you have (network) access to { jenkinsDetails.Url }. URLs the agent need communicate with - { BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform() }."
});
return;
}
catch (Exception ex)
{
context.AddIssue(new Issue {
Type = IssueType.Warning, Message = StringUtil.Loc("DownloadingJenkinsCommitsFailedWithException", jenkinsDetails.Alias, ex.ToString())
});
return;
}
}
else
{
context.AddIssue(new Issue {
Type = IssueType.Warning, Message = StringUtil.Loc("JenkinsCommitsInvalidEndJobId", jenkinsDetails.EndCommitArtifactVersion, jenkinsDetails.Alias)
});
return;
}
}
else
{
context.Debug("No commit details found in the agent artifact. Not downloading the commits");
}
}
private async Task DownloadArtifactAsync(
IExecutionContext executionContext,
ServerBuildArtifact buildArtifact,
ArtifactDefinition artifactDefinition,
string localFolderPath,
BuildHttpClient buildClient,
XamlBuildHttpClient xamlBuildClient,
DefinitionType definitionType,
int buildId)
{
var downloadFolderPath = Path.Combine(localFolderPath, buildArtifact.Name);
var buildArtifactDetails = artifactDefinition.Details as BuildArtifactDetails;
if ((buildArtifact.Resource.Type == null && buildArtifact.Id == 0) || // bug on build API Bug 378900
string.Equals(buildArtifact.Resource.Type, WellKnownArtifactResourceTypes.FilePath, StringComparison.OrdinalIgnoreCase))
{
executionContext.Output("Artifact Type: FileShare");
string fileShare;
if (buildArtifact.Id == 0)
{
fileShare = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath;
}
else
{
fileShare = new Uri(Path.Combine(buildArtifact.Resource.DownloadUrl, buildArtifact.Name)).LocalPath;
if (!Directory.Exists(fileShare))
{
// download path does not exist, log and fall back
executionContext.Output(StringUtil.Loc("RMArtifactNameDirectoryNotFound", fileShare, buildArtifact.Resource.DownloadUrl));
fileShare = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath;
}
}
if (!Directory.Exists(fileShare))
{
// download path does not exist, raise exception
throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactDirectoryNotFoundError", fileShare));
}
var fileShareArtifact = new FileShareArtifact();
await fileShareArtifact.DownloadArtifactAsync(executionContext, HostContext, artifactDefinition, fileShare, downloadFolderPath);
}
else if (string.Equals(buildArtifact.Resource.Type, WellKnownArtifactResourceTypes.Container, StringComparison.OrdinalIgnoreCase))
{
executionContext.Output("Artifact Type: ServerDrop");
// TODO:Get VssBinFetchclient and get away from zipstream downloader
Stream contentStream;
if (definitionType == DefinitionType.Xaml)
{
contentStream = await xamlBuildClient.GetArtifactContentZipAsync(buildArtifactDetails.Project, buildId, buildArtifact.Name);
}
else
{
contentStream = await buildClient.GetArtifactContentZipAsync(buildArtifactDetails.Project, buildId, buildArtifact.Name);
}
var zipStreamDownloader = HostContext.GetService <IZipStreamDownloader>();
string artifactRootFolder = StringUtil.Format("/{0}", buildArtifact.Name);
await zipStreamDownloader.DownloadFromStream(contentStream, artifactRootFolder, buildArtifactDetails.RelativePath, downloadFolderPath);
}
else
{
executionContext.Warning(StringUtil.Loc("RMArtifactTypeNotSupported", buildArtifact.Resource.Type));
}
}
public async Task DownloadAsync(
IExecutionContext executionContext,
ArtifactDefinition artifactDefinition,
string localFolderPath)
{
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath));
var jenkinsDetails = artifactDefinition.Details as JenkinsArtifactDetails;
executionContext.Output(StringUtil.Loc("RMGotJenkinsArtifactDetails"));
executionContext.Output(StringUtil.Loc("RMJenkinsJobName", jenkinsDetails.JobName));
executionContext.Output(StringUtil.Loc("RMJenkinsBuildId", jenkinsDetails.BuildId));
IGenericHttpClient client = HostContext.GetService <IGenericHttpClient>();
if (!IsValidBuild(client, jenkinsDetails))
{
throw new ArtifactDownloadException(StringUtil.Loc("RMJenkinsInvalidBuild", jenkinsDetails.BuildId));
}
Stream downloadedStream = null;
string downloadArtifactsUrl =
string.Format(
CultureInfo.InvariantCulture,
"{0}/job/{1}/{2}/artifact/{3}/*zip*/",
jenkinsDetails.Url,
jenkinsDetails.JobName,
jenkinsDetails.BuildId,
jenkinsDetails.RelativePath);
executionContext.Output(StringUtil.Loc("RMPrepareToGetFromJenkinsServer"));
HttpResponseMessage response = client.GetAsync(downloadArtifactsUrl, jenkinsDetails.AccountName, jenkinsDetails.AccountPassword, jenkinsDetails.AcceptUntrustedCertificates).Result;
if (response.IsSuccessStatusCode)
{
downloadedStream = response.Content.ReadAsStreamAsync().Result;
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
executionContext.Warning(StringUtil.Loc("RMJenkinsNoArtifactsFound", jenkinsDetails.BuildId));
return;
}
else
{
throw new ArtifactDownloadException(StringUtil.Loc("RMDownloadArtifactUnexpectedError"));
}
var parentFolder = GetParentFolderName(jenkinsDetails.RelativePath);
Trace.Info($"Found parentFolder {parentFolder} for relative path {jenkinsDetails.RelativePath}");
executionContext.Output(StringUtil.Loc("RMDownloadingJenkinsArtifacts"));
var zipStreamDownloader = HostContext.GetService <IZipStreamDownloader>();
await zipStreamDownloader.DownloadFromStream(
executionContext,
downloadedStream,
string.IsNullOrEmpty(parentFolder)? "archive" : string.Empty,
parentFolder,
localFolderPath);
}
public async Task DownloadAsync(IExecutionContext executionContext, ArtifactDefinition artifactDefinition, string localFolderPath)
{
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath));
int buildId = Convert.ToInt32(artifactDefinition.Version, CultureInfo.InvariantCulture);
if (buildId <= 0)
{
throw new ArgumentException("artifactDefinition.Version");
}
var buildArtifactDetails = artifactDefinition.Details as BuildArtifactDetails;
if (buildArtifactDetails == null)
{
throw new ArgumentException("artifactDefinition.Details");
}
// Get the list of available artifacts from build.
executionContext.Output(StringUtil.Loc("RMPreparingToGetBuildArtifactList"));
var vssConnection = VssUtil.CreateConnection(buildArtifactDetails.TfsUrl, buildArtifactDetails.Credentials);
var buildClient = vssConnection.GetClient <BuildHttpClient>();
var xamlBuildClient = vssConnection.GetClient <XamlBuildHttpClient>();
List <ServerBuildArtifact> buildArtifacts = null;
EnsureVersionBelongsToLinkedDefinition(artifactDefinition, buildClient, xamlBuildClient);
try
{
buildArtifacts = await buildClient.GetArtifactsAsync(buildArtifactDetails.Project, buildId);
}
catch (BuildNotFoundException)
{
buildArtifacts = await xamlBuildClient.GetArtifactsAsync(buildArtifactDetails.Project, buildId);
}
// No artifacts found in the build, add warning.
if (buildArtifacts == null || !buildArtifacts.Any())
{
executionContext.Warning(StringUtil.Loc("RMNoBuildArtifactsFound", buildId));
return;
}
// DownloadFromStream each of the artifact sequentially.
// TODO: Should we download them parallely?
foreach (ServerBuildArtifact buildArtifact in buildArtifacts)
{
if (Match(buildArtifact, artifactDefinition))
{
executionContext.Output(StringUtil.Loc("RMPreparingToDownload", buildArtifact.Name));
await this.DownloadArtifactAsync(executionContext, buildArtifact, artifactDefinition, localFolderPath);
}
else
{
executionContext.Warning(StringUtil.Loc("RMArtifactMatchNotFound", buildArtifact.Name));
}
}
}
public async Task DownloadCommitsAsync(IExecutionContext context, ArtifactDefinition artifactDefinition, string commitsWorkFolder)
{
Trace.Entering();
var jenkinsDetails = artifactDefinition.Details as JenkinsArtifactDetails;
int startJobId = 0, endJobId = 0;
if (!string.IsNullOrEmpty(jenkinsDetails.EndCommitArtifactVersion))
{
if (int.TryParse(jenkinsDetails.EndCommitArtifactVersion, out endJobId))
{
context.Output(StringUtil.Loc("RMDownloadingCommits"));
if (int.TryParse(jenkinsDetails.StartCommitArtifactVersion, out startJobId))
{
if (startJobId < endJobId)
{
context.Output(StringUtil.Loc("DownloadingJenkinsCommitsBetween", startJobId, endJobId));
}
else if (startJobId > endJobId)
{
context.Output(StringUtil.Loc("JenkinsRollbackDeployment", startJobId, endJobId));
// swap the job IDs to fetch the roll back commits
int swap = startJobId;
startJobId = endJobId;
endJobId = swap;
}
else if (startJobId == endJobId)
{
context.Output(StringUtil.Loc("JenkinsNoCommitsToFetch"));
return;
}
}
else
{
context.Debug(StringUtil.Loc("JenkinsDownloadingChangeFromCurrentBuild"));
}
try
{
IEnumerable <Change> changes = await DownloadCommits(context, jenkinsDetails, startJobId, endJobId);
if (changes.Any())
{
string commitsFileName = GetCommitsFileName(jenkinsDetails.Alias);
string commitsFilePath = Path.Combine(commitsWorkFolder, commitsFileName);
context.Debug($"Commits will be written to {commitsFilePath}");
WriteCommitsToFile(context, changes, commitsFilePath);
context.Debug($"Commits written to {commitsFilePath}");
context.QueueAttachFile(CoreAttachmentType.FileAttachment, commitsFileName, commitsFilePath);
}
}
catch (Exception ex)
{
context.AddIssue(new Issue {
Type = IssueType.Warning, Message = StringUtil.Loc("DownloadingJenkinsCommitsFailedWithException", jenkinsDetails.Alias, ex.ToString())
});
return;
}
}
else
{
context.AddIssue(new Issue {
Type = IssueType.Warning, Message = StringUtil.Loc("JenkinsCommitsInvalidEndJobId", jenkinsDetails.EndCommitArtifactVersion, jenkinsDetails.Alias)
});
return;
}
}
else
{
context.Debug("No commit details found in the agent artifact. Not downloading the commits");
}
}
// This is only used by build artifact. This isn't a officially supported artifact type in RM
public async Task DownloadArtifactAsync(IExecutionContext executionContext, IHostContext hostContext, ArtifactDefinition artifactDefinition, string dropLocation, string localFolderPath)
{
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath));
ArgUtil.NotNullOrEmpty(dropLocation, nameof(dropLocation));
bool disableRobocopy = executionContext.Variables.GetBoolean(Constants.Variables.Release.DisableRobocopy) ?? false;
if (disableRobocopy == false)
{
await DownloadArtifactUsingRobocopyAsync(executionContext, hostContext, artifactDefinition, dropLocation, localFolderPath);
}
else
{
await DownloadArtifactUsingFileSystemManagerAsync(executionContext, hostContext, artifactDefinition, dropLocation, localFolderPath);
}
}
// This is only used by build artifact. This isn't a officially supported artifact type in RM
public async Task DownloadArtifactAsync(IExecutionContext executionContext, IHostContext hostContext, ArtifactDefinition artifactDefinition, string dropLocation, string localFolderPath)
{
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath));
ArgUtil.NotNullOrEmpty(dropLocation, nameof(dropLocation));
var trimChars = new[] { '\\', '/' };
var relativePath = artifactDefinition.Details.RelativePath;
// If user has specified a relative folder in the drop, change the drop location itself.
dropLocation = Path.Combine(dropLocation.TrimEnd(trimChars), relativePath.Trim(trimChars));
var fileSystemManager = hostContext.CreateService <IReleaseFileSystemManager>();
List <string> filePaths =
new DirectoryInfo(dropLocation).EnumerateFiles("*", SearchOption.AllDirectories)
.Select(path => path.FullName)
.ToList();
if (filePaths.Any())
{
foreach (var filePath in filePaths)
{
string fullPath = Path.GetFullPath(filePath);
if (File.Exists(fullPath))
{
string filePathRelativeToDrop = filePath.Replace(dropLocation, string.Empty).Trim(trimChars);
using (StreamReader fileReader = fileSystemManager.GetFileReader(filePath))
{
await
fileSystemManager.WriteStreamToFile(
fileReader.BaseStream,
Path.Combine(localFolderPath, filePathRelativeToDrop),
executionContext.CancellationToken);
}
}
else
{
executionContext.Warning(StringUtil.Loc("FileNotFound", fullPath));
}
}
}
else
{
executionContext.Warning(StringUtil.Loc("RMArtifactEmpty"));
}
}
private async Task DownloadArtifactUsingRobocopyAsync(IExecutionContext executionContext, IHostContext hostContext, ArtifactDefinition artifactDefinition, string dropLocation, string downloadFolderPath)
{
int? robocopyMT = executionContext.Variables.GetInt(Constants.Variables.Release.RobocopyMT);
bool verbose = executionContext.Variables.GetBoolean(Constants.Variables.System.Debug) ?? false;
if (robocopyMT != null)
{
if (robocopyMT < 1)
{
robocopyMT = 1;
}
else if (robocopyMT > 128)
{
robocopyMT = 128;
}
}
executionContext.Output(StringUtil.Loc("RMDownloadingArtifactUsingRobocopy"));
using (var processInvoker = hostContext.CreateService <IProcessInvoker>())
{
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
executionContext.Output(stdout.Data);
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
executionContext.Error(stderr.Data);
}
};
var trimChars = new[] { '\\', '/' };
var relativePath = artifactDefinition.Details.RelativePath;
dropLocation = Path.Combine(dropLocation.TrimEnd(trimChars), relativePath.Trim(trimChars));
downloadFolderPath = downloadFolderPath.TrimEnd(trimChars);
string robocopyArguments = "\"" + dropLocation + "\" \"" + downloadFolderPath + "\" /E /Z /NP /R:3";
if (verbose != true)
{
robocopyArguments = robocopyArguments + " /NDL /NFL";
}
if (robocopyMT != null)
{
robocopyArguments = robocopyArguments + " /MT:" + robocopyMT;
}
int exitCode = await processInvoker.ExecuteAsync(
workingDirectory : "",
fileName : "robocopy",
arguments : robocopyArguments,
environment : null,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : true,
cancellationToken : executionContext.CancellationToken);
executionContext.Output(StringUtil.Loc("RMRobocopyBasedArtifactDownloadExitCode", exitCode));
if (exitCode >= 8)
{
throw new ArtifactDownloadException(StringUtil.Loc("RMRobocopyBasedArtifactDownloadFailed", exitCode));
}
}
}
