public async Task <ISentJob> SendAsync(Action <string> log = null)
{
IBlobHelper storage;
if (string.IsNullOrEmpty(StorageAccountConnectionString))
{
storage = new ApiBlobHelper(HelixApi.Storage);
}
else
{
storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
}
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);
var jobList = new List <JobListEntry>();
List <string> correlationPayloadUris =
(await Task.WhenAll(CorrelationPayloads.Select(p => p.UploadAsync(storageContainer, log)))).ToList();
jobList = (await Task.WhenAll(
_workItems.Select(async w =>
{
var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
entry.CorrelationPayloadUris = correlationPayloadUris;
return(entry);
}
))).ToList();
string jobListJson = JsonConvert.SerializeObject(jobList);
Uri jobListUri = await storageContainer.UploadTextAsync(
jobListJson,
$"job-list-{Guid.NewGuid()}.json");
string jobStartIdentifier = Guid.NewGuid().ToString("N");
JobCreationResult newJob = await HelixApi.RetryAsync(
() => JobApi.NewAsync(
new JobCreationRequest(
Source,
Type,
Build,
_properties.ToImmutableDictionary(),
jobListUri.ToString(),
TargetQueueId,
storageContainer.Uri,
storageContainer.ReadSas,
storageContainer.WriteSas)
{
Creator = Creator,
MaxRetryCount = MaxRetryCount ?? 0,
JobStartIdentifier = jobStartIdentifier,
}),
ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));
return(new SentJob(JobApi, newJob));
}
public bool IsTransient(Exception ex)
{
if (ex is HttpRequestWithStatusException)
{
return(_httpStatusCodeErrorDetectionStrategy.IsTransient(ex));
}
return(HelixApi.IsRetryableHttpException(ex));
}
public PagedResponse(HelixApi client, Func <HttpRequestMessage, HttpResponseMessage, Task> onFailure, IHttpOperationResponse <IImmutableList <T> > response)
{
_onFailure = onFailure;
Client = client;
Values = response.Body;
if (!response.Response.Headers.TryGetValues("Link", out var linkHeader))
{
return;
}
var links = ParseLinkHeader(linkHeader).ToList();
FirstPageLink = links.FirstOrDefault(t => t.rel == "first").href;
PrevPageLink = links.FirstOrDefault(t => t.rel == "prev").href;
NextPageLink = links.FirstOrDefault(t => t.rel == "next").href;
LastPageLink = links.FirstOrDefault(t => t.rel == "last").href;
}
public PagedResponse(HelixApi client, Func <Request, Response, Task> onFailure, Response response, IImmutableList <T> values)
{
_onFailure = onFailure;
Client = client;
Values = values;
if (!response.Headers.TryGetValues("Link", out var linkHeader))
{
return;
}
var links = ParseLinkHeader(linkHeader).ToList();
FirstPageLink = links.FirstOrDefault(t => t.rel == "first").href;
PrevPageLink = links.FirstOrDefault(t => t.rel == "prev").href;
NextPageLink = links.FirstOrDefault(t => t.rel == "next").href;
LastPageLink = links.FirstOrDefault(t => t.rel == "last").href;
}
public async Task <ISentJob> SendAsync(Action <string> log = null)
{
IBlobHelper storage;
if (string.IsNullOrEmpty(StorageAccountConnectionString))
{
storage = new ApiBlobHelper(HelixApi.Storage);
}
else
{
storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
}
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);
var jobList = new List <JobListEntry>();
IBlobContainer resultsStorageContainer = null;
if (!string.IsNullOrEmpty(ResultsStorageAccountConnectionString))
{
IBlobHelper resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString);
resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetResultsContainerName);
}
else if (_withDefaultResultsContainer)
{
resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName);
}
Dictionary <string, string> correlationPayloadUris =
(await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);
jobList = (await Task.WhenAll(
_workItems.Select(async w =>
{
var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
return(entry);
}
))).ToList();
string jobListJson = JsonConvert.SerializeObject(jobList);
Uri jobListUri = await storageContainer.UploadTextAsync(
jobListJson,
$"job-list-{Guid.NewGuid()}.json");
// Don't log the sas, remove the query string.
string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");
log?.Invoke($"Created job list at {jobListUriForLogging}");
string jobStartIdentifier = Guid.NewGuid().ToString("N");
JobCreationResult newJob = await HelixApi.RetryAsync(
() => JobApi.NewAsync(
new JobCreationRequest(
Source,
Type,
Build,
_properties.ToImmutableDictionary(),
jobListUri.ToString(),
TargetQueueId)
{
Creator = Creator,
MaxRetryCount = MaxRetryCount ?? 0,
JobStartIdentifier = jobStartIdentifier,
ResultsUri = resultsStorageContainer?.Uri,
ResultsUriRSAS = resultsStorageContainer?.ReadSas,
ResultsUriWSAS = resultsStorageContainer?.WriteSas,
}),
ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));
return(new SentJob(JobApi, newJob, resultsStorageContainer?.Uri, string.IsNullOrEmpty(Creator) ? resultsStorageContainer?.ReadSas : string.Empty));
}
public async Task <ISentJob> SendAsync(Action <string> log = null)
{
IBlobHelper storage;
if (string.IsNullOrEmpty(StorageAccountConnectionString))
{
storage = new ApiBlobHelper(HelixApi.Storage);
}
else
{
storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
}
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);
var jobList = new List <JobListEntry>();
IBlobContainer resultsStorageContainer = null;
if (!string.IsNullOrEmpty(ResultsStorageAccountConnectionString))
{
IBlobHelper resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString);
resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetResultsContainerName);
}
else if (_withDefaultResultsContainer)
{
resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName);
}
Dictionary <string, string> correlationPayloadUris =
(await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);
jobList = (await Task.WhenAll(
_workItems.Select(async w =>
{
var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
return(entry);
}
))).ToList();
string jobListJson = JsonConvert.SerializeObject(jobList);
Uri jobListUri = await storageContainer.UploadTextAsync(
jobListJson,
$"job-list-{Guid.NewGuid()}.json");
// Don't log the sas, remove the query string.
string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");
log?.Invoke($"Created job list at {jobListUriForLogging}");
// Only specify the ResultContainerPrefix if both repository name and source branch are available.
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH")))
{
// Container names can only be alphanumeric (plus dashes) lowercase names, with no consecutive dashes.
// Replace / with -, make all branch and repository names lowercase, remove any characters not
// allowed in container names, and replace any string of dashes with a single dash.
Regex illegalCharacters = new Regex("[^a-z0-9-]");
Regex multipleDashes = new Regex("-{2,}");
string repoName = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME");
string branchName = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");
// ResultContainerPrefix will be <Repository Name>-<BranchName>
ResultContainerPrefix = $"{repoName}-{branchName}-".Replace("/", "-").ToLower();
ResultContainerPrefix = multipleDashes.Replace(illegalCharacters.Replace(ResultContainerPrefix, ""), "-");
}
string jobStartIdentifier = Guid.NewGuid().ToString("N");
JobCreationResult newJob = await HelixApi.RetryAsync(
() => JobApi.NewAsync(
new JobCreationRequest(
Source,
Type,
Build,
_properties.ToImmutableDictionary(),
jobListUri.ToString(),
TargetQueueId)
{
Creator = Creator,
MaxRetryCount = MaxRetryCount ?? 0,
JobStartIdentifier = jobStartIdentifier,
ResultsUri = resultsStorageContainer?.Uri,
ResultsUriRSAS = resultsStorageContainer?.ReadSas,
ResultsUriWSAS = resultsStorageContainer?.WriteSas,
ResultContainerPrefix = ResultContainerPrefix,
}),
ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));
return(new SentJob(JobApi, newJob, resultsStorageContainer?.Uri, string.IsNullOrEmpty(Creator) ? resultsStorageContainer?.ReadSas : string.Empty));
}
public async Task <ISentJob> SendAsync(Action <string> log = null)
{
IBlobHelper storage;
if (string.IsNullOrEmpty(StorageAccountConnectionString))
{
storage = new ApiBlobHelper(HelixApi.Storage);
}
else
{
storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
}
var(queueId, dockerTag, queueAlias) = ParseQueueId(TargetQueueId);
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName, queueId);
var jobList = new List <JobListEntry>();
Dictionary <string, string> correlationPayloadUris =
(await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);
jobList = (await Task.WhenAll(
_workItems.Select(async w =>
{
var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
return(entry);
}
))).ToList();
string jobListJson = JsonConvert.SerializeObject(jobList);
Uri jobListUri = await storageContainer.UploadTextAsync(
jobListJson,
$"job-list-{Guid.NewGuid()}.json");
// Don't log the sas, remove the query string.
string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");
log?.Invoke($"Created job list at {jobListUriForLogging}");
// Only specify the ResultContainerPrefix if both repository name and source branch are available.
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH")))
{
// Container names can only be alphanumeric (plus dashes) lowercase names, with no consecutive dashes.
// Replace / with -, make all branch and repository names lowercase, remove any characters not
// allowed in container names, and replace any string of dashes with a single dash.
Regex illegalCharacters = new Regex("[^a-z0-9-]");
Regex multipleDashes = new Regex("-{2,}");
string repoName = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME");
string branchName = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");
// ResultContainerPrefix will be <Repository Name>-<BranchName>
ResultContainerPrefix = $"{repoName}-{branchName}-".Replace("/", "-").ToLower();
ResultContainerPrefix = multipleDashes.Replace(illegalCharacters.Replace(ResultContainerPrefix, ""), "-");
}
var creationRequest = new JobCreationRequest(Type, _properties.ToImmutableDictionary(), jobListUri.ToString(), queueId)
{
Creator = Creator,
ResultContainerPrefix = ResultContainerPrefix,
DockerTag = dockerTag,
QueueAlias = queueAlias,
};
if (string.IsNullOrEmpty(Source))
{
// We only want to specify a branch if Source wasn't already provided.
// Latest Helix Job API will 400 if both Source and any of SourcePrefix, TeamProject, Repository, or Branch are set.
InitializeSourceParameters(creationRequest);
}
else
{
creationRequest.Source = Source;
}
string jobStartIdentifier = Guid.NewGuid().ToString("N");
JobCreationResult newJob = await HelixApi.RetryAsync(
() => JobApi.NewAsync(creationRequest, jobStartIdentifier),
ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."),
IsRetryableJobListUriHttpError,
CancellationToken.None);
return(new SentJob(JobApi, newJob, newJob.ResultsUri, newJob.ResultsUriRSAS));
}
