private async Task DownloadCommits(
IExecutionContext executionContext,
Guid teamProjectId,
IList <AgentArtifactDefinition> agentArtifactDefinitions)
{
Trace.Entering();
Trace.Info("Creating commit work folder");
string commitsWorkFolder = GetCommitsWorkFolder(executionContext);
// Note: We are having an explicit type here. For other artifact types we are planning to go with tasks
// Only for jenkins we are making the agent to download
var extensionManager = HostContext.GetService <IExtensionManager>();
JenkinsArtifact jenkinsExtension = (extensionManager.GetExtensions <IArtifactExtension>()).FirstOrDefault(x => x.ArtifactType == AgentArtifactType.Jenkins) as JenkinsArtifact;
foreach (AgentArtifactDefinition agentArtifactDefinition in agentArtifactDefinitions)
{
if (agentArtifactDefinition.ArtifactType == AgentArtifactType.Jenkins)
{
Trace.Info($"Found supported artifact {agentArtifactDefinition.Alias} for downloading commits");
ArtifactDefinition artifactDefinition = ConvertToArtifactDefinition(agentArtifactDefinition, executionContext, jenkinsExtension);
await jenkinsExtension.DownloadCommitsAsync(executionContext, artifactDefinition, commitsWorkFolder);
}
}
}
public async void CommitsShouldBeUploadedAsAttachment()
{
using (TestHostContext tc = Setup())
{
string commitRootDirectory = Path.Combine(tc.GetDirectory(WellKnownDirectory.Work), Guid.NewGuid().ToString("D"));
Directory.CreateDirectory(commitRootDirectory);
try
{
JenkinsArtifactDetails details = _artifactDefinition.Details as JenkinsArtifactDetails;
details.StartCommitArtifactVersion = "10";
details.EndCommitArtifactVersion = "20";
var artifact = new JenkinsArtifact();
artifact.Initialize(tc);
SetupBuildRangeQuery(details, "{ \"allBuilds\": [{ \"number\": 20 }, { \"number\": 10 }, { \"number\": 2 } ] }");
string commitResult = " {\"builds\": [{ \"number\":9, \"result\":\"SUCCESS\", \"changeSet\": { \"items\": [{ \"commitId\" : \"2869c7ccd0b1b649ba6765e89ee5ff36ef6d4805\", \"author\": { \"fullName\" : \"testuser\" }, \"msg\":\"test\" }]}}]}";
string commitsUrl = $"{details.Url}/job/{details.JobName}/api/json?tree=builds[number,result,changeSet[items[commitId,date,msg,author[fullName]]]]{{0,1}}";
_httpClient.Setup(x => x.GetStringAsync(It.Is <string>(y => y.StartsWith(commitsUrl)), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>()))
.Returns(Task.FromResult(commitResult));
string commitFilePath = Path.Combine(commitRootDirectory, $"commits_{details.Alias}_1.json");
await artifact.DownloadCommitsAsync(_ec.Object, _artifactDefinition, commitRootDirectory);
_ec.Verify(x => x.QueueAttachFile(It.Is <string>(y => y.Equals(CoreAttachmentType.FileAttachment)), It.IsAny <string>(), It.IsAny <string>()), Times.Once);
}
finally
{
IOUtil.DeleteDirectory(commitRootDirectory, CancellationToken.None);
}
}
}
/// <summary>
/// Save to a file.
/// </summary>
/// <param name="?"></param>
/// <param name="destination"></param>
/// <returns></returns>
public static async Task <FileInfo> SaveToFile(this JenkinsArtifact artifact, JenkinsBuild bld, FileInfo destination)
{
var url = string.Format("{0}artifact/{1}", bld.Url, artifact.RelativePath);
var fileData = await TDTJobAccess.GetAsBytes(url).FirstAsync();
using (var wr = destination.Create())
{
wr.Write(fileData, 0, fileData.Length);
}
return(destination);
}
public async void IfNoCommitVersionExistsInArtifactDetailsNoIssueShouldBeAdded()
{
using (TestHostContext tc = Setup())
{
var trace = tc.GetTrace();
var artifact = new JenkinsArtifact();
artifact.Initialize(tc);
await artifact.DownloadCommitsAsync(_ec.Object, _artifactDefinition, "test");
_ec.Verify(x => x.AddIssue(It.Is <Issue>(y => y.Type == IssueType.Warning)), Times.Never);
}
}
public async void MissingStartVersionShouldDownloadCommitsFromSingleBuild()
{
using (TestHostContext tc = Setup())
{
JenkinsArtifactDetails details = _artifactDefinition.Details as JenkinsArtifactDetails;
details.EndCommitArtifactVersion = "10";
var artifact = new JenkinsArtifact();
artifact.Initialize(tc);
string expectedUrl = $"{details.Url}/job/{details.JobName}/{details.EndCommitArtifactVersion}/api/json?tree=number,result,changeSet[items[commitId,date,msg,author[fullName]]]";
await artifact.DownloadCommitsAsync(_ec.Object, _artifactDefinition, tc.GetDirectory(WellKnownDirectory.Root));
_httpClient.Verify(x => x.GetStringAsync(It.Is <string>(y => y.StartsWith(expectedUrl)), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>()), Times.Once);
}
}
public async void ShouldLogAnIssueIfEndVersionIsInvalidInArtifactDetail()
{
using (TestHostContext tc = Setup())
{
var trace = tc.GetTrace();
JenkinsArtifactDetails details = _artifactDefinition.Details as JenkinsArtifactDetails;
details.EndCommitArtifactVersion = "xx";
var artifact = new JenkinsArtifact();
artifact.Initialize(tc);
await artifact.DownloadCommitsAsync(_ec.Object, _artifactDefinition, "test");
_ec.Verify(x => x.AddIssue(It.Is <Issue>(y => y.Type == IssueType.Warning)), Times.Once);
}
}
public async void JenkinsCommitsShouldLogAnIssueIfBuildIsDeleted()
{
using (TestHostContext tc = Setup())
{
JenkinsArtifactDetails details = _artifactDefinition.Details as JenkinsArtifactDetails;
details.StartCommitArtifactVersion = "10";
details.EndCommitArtifactVersion = "20";
var artifact = new JenkinsArtifact();
artifact.Initialize(tc);
SetupBuildRangeQuery(details, "{ \"allBuilds\": [{ \"number\": 30 }, { \"number\": 29 }, { \"number\": 28 } ] }");
await artifact.DownloadCommitsAsync(_ec.Object, _artifactDefinition, tc.GetDirectory(WellKnownDirectory.Root));
_ec.Verify(x => x.AddIssue(It.Is <Issue>(y => y.Type == IssueType.Warning)), Times.Once);
}
}
private Metadata TransformOne(Metadata metadata, JObject json, JenkinsBuild build, JenkinsOptions options)
{
JenkinsArtifact[] artifacts = build.Artifacts
.Where(a => options.AssetMatchPattern.IsMatch(a.FileName))
.ToArray();
switch (artifacts.Length)
{
case 1:
JenkinsArtifact artifact = artifacts.Single();
string download = Uri.EscapeUriString(
$"{build.Url}artifact/{artifact.RelativePath}"
);
Log.DebugFormat("Using download URL: {0}", download);
json.SafeAdd("download", download);
if (options.UseFilenameVersion)
{
Log.DebugFormat("Using filename as version: {0}", artifact.FileName);
json.SafeAdd("version", artifact.FileName);
}
// Make sure resources exist.
if (json["resources"] == null)
{
json["resources"] = new JObject();
}
var resourcesJson = (JObject)json["resources"];
resourcesJson.SafeAdd("ci", Uri.EscapeUriString(metadata.Kref.Id));
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
return(new Metadata(json));
break;
case 0:
throw new Exception("Could not find any matching artifacts");
default:
throw new Exception("Found too many matching artifacts");
}
}
public async void JenkinsCommitsShouldBeFetchedBetweenBuildRange()
{
using (TestHostContext tc = Setup())
{
JenkinsArtifactDetails details = _artifactDefinition.Details as JenkinsArtifactDetails;
details.StartCommitArtifactVersion = "10";
details.EndCommitArtifactVersion = "20";
var artifact = new JenkinsArtifact();
artifact.Initialize(tc);
SetupBuildRangeQuery(details, "{ \"allBuilds\": [{ \"number\": 20 }, { \"number\": 10 }, { \"number\": 2 } ] }");
string expectedUrl = $"{details.Url}/job/{details.JobName}/api/json?tree=builds[number,result,changeSet[items[commitId,date,msg,author[fullName]]]]{{0,1}}";
await artifact.DownloadCommitsAsync(_ec.Object, _artifactDefinition, tc.GetDirectory(WellKnownDirectory.Root));
_httpClient.Verify(x => x.GetStringAsync(It.Is <string>(y => y.StartsWith(expectedUrl)), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>()), Times.Once);
}
}
public async void CommitsShoulHaveUrlIfItsGitRepo()
{
using (TestHostContext tc = Setup())
{
string commitRootDirectory = tc.GetDirectory(WellKnownDirectory.Root);
try
{
JenkinsArtifactDetails details = _artifactDefinition.Details as JenkinsArtifactDetails;
details.StartCommitArtifactVersion = "10";
details.EndCommitArtifactVersion = "20";
var artifact = new JenkinsArtifact();
artifact.Initialize(tc);
SetupBuildRangeQuery(details, "{ \"allBuilds\": [{ \"number\": 20 }, { \"number\": 10 }, { \"number\": 2 } ] }");
string commitResult = " {\"builds\": [{ \"number\":9, \"result\":\"SUCCESS\", \"changeSet\": { \"items\": [{ \"commitId\" : \"2869c7ccd0b1b649ba6765e89ee5ff36ef6d4805\", \"author\": { \"fullName\" : \"testuser\" }, \"msg\":\"test\" }]}}]}";
string commitsUrl = $"{details.Url}/job/{details.JobName}/api/json?tree=builds[number,result,changeSet[items[commitId,date,msg,author[fullName]]]]{{0,1}}";
_httpClient.Setup(x => x.GetStringAsync(It.Is <string>(y => y.StartsWith(commitsUrl)), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>()))
.Returns(Task.FromResult(commitResult));
string repoUrl = $"{details.Url}/job/{details.JobName}/{details.EndCommitArtifactVersion}/api/json?tree=actions[remoteUrls],changeSet[kind]";
string repoResult = "{ \"actions\": [ { \"remoteUrls\": [ \"https://github.com/TestUser/TestRepo\" ] }, ], \"changeSet\": { \"kind\": \"git\" } }";
_httpClient.Setup(x => x.GetStringAsync(It.Is <string>(y => y.StartsWith(repoUrl)), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>()))
.Returns(Task.FromResult(repoResult));
string commitFilePath = Path.Combine(commitRootDirectory, $"commits_{details.Alias}_1.json");
Directory.CreateDirectory(commitRootDirectory);
string expectedCommitUrl = "https://github.com/TestUser/TestRepo/commit/2869c7ccd0b1b649ba6765e89ee5ff36ef6d4805";
await artifact.DownloadCommitsAsync(_ec.Object, _artifactDefinition, commitRootDirectory);
_ec.Verify(x => x.QueueAttachFile(It.Is <string>(y => y.Equals(CoreAttachmentType.FileAttachment)), It.IsAny <string>(), It.Is <string>(z => string.Join("", File.ReadAllLines(z)).Contains(expectedCommitUrl))), Times.Once);
}
finally
{
IOUtil.DeleteDirectory(commitRootDirectory, CancellationToken.None);
}
}
}
public Metadata Transform(Metadata metadata)
{
if (metadata.Kref != null && metadata.Kref.Source == "jenkins")
{
var json = metadata.Json();
Log.InfoFormat("Executing Jenkins transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
JenkinsOptions options = json["x_netkan_jenkins"]?.ToObject <JenkinsOptions>()
?? new JenkinsOptions();
JenkinsBuild build = _api.GetLatestBuild(
new JenkinsRef(metadata.Kref),
options
);
JenkinsArtifact[] artifacts = build.Artifacts
.Where(a => options.AssetMatchPattern.IsMatch(a.FileName))
.ToArray();
switch (artifacts.Length)
{
case 1:
JenkinsArtifact artifact = artifacts.Single();
string download = Uri.EscapeUriString(
$"{build.Url}artifact/{artifact.RelativePath}"
);
Log.DebugFormat("Using download URL: {0}", download);
json.SafeAdd("download", download);
if (options.UseFilenameVersion)
{
Log.DebugFormat("Using filename as version: {0}", artifact.FileName);
json.SafeAdd("version", artifact.FileName);
}
// Make sure resources exist.
if (json["resources"] == null)
{
json["resources"] = new JObject();
}
var resourcesJson = (JObject)json["resources"];
resourcesJson.SafeAdd("ci", Uri.EscapeUriString(metadata.Kref.Id));
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
return(new Metadata(json));
break;
case 0:
throw new Exception("Could not find any matching artifacts");
default:
throw new Exception("Found too many matching artifacts");
}
}
return(metadata);
}
/// <summary>
/// Download a file to a temp location.
/// </summary>
/// <param name="artifact"></param>
/// <param name="extension">The extension the file shoudl have, without the dot.</param>
/// <returns></returns>
public static Task <FileInfo> SaveToTempFile(this JenkinsArtifact artifact, JenkinsBuild bld, string extension)
{
var fname = string.Format("{0}{1}.{2}", Path.GetTempPath(), Guid.NewGuid(), extension);
return(artifact.SaveToFile(bld, new FileInfo(fname)));
}
