public async Task <ISentJob> SendAsync(Action <string> log = null)
{
IBlobHelper storage;
if (string.IsNullOrEmpty(StorageAccountConnectionString))
{
storage = new ApiBlobHelper(HelixApi.Storage);
}
else
{
storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
}
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);
var jobList = new List <JobListEntry>();
List <string> correlationPayloadUris =
(await Task.WhenAll(CorrelationPayloads.Select(p => p.UploadAsync(storageContainer, log)))).ToList();
jobList = (await Task.WhenAll(
_workItems.Select(async w =>
{
var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
entry.CorrelationPayloadUris = correlationPayloadUris;
return(entry);
}
))).ToList();
string jobListJson = JsonConvert.SerializeObject(jobList);
Uri jobListUri = await storageContainer.UploadTextAsync(
jobListJson,
$"job-list-{Guid.NewGuid()}.json");
string jobStartIdentifier = Guid.NewGuid().ToString("N");
JobCreationResult newJob = await HelixApi.RetryAsync(
() => JobApi.NewAsync(
new JobCreationRequest(
Source,
Type,
Build,
_properties.ToImmutableDictionary(),
jobListUri.ToString(),
TargetQueueId,
storageContainer.Uri,
storageContainer.ReadSas,
storageContainer.WriteSas)
{
Creator = Creator,
MaxRetryCount = MaxRetryCount ?? 0,
JobStartIdentifier = jobStartIdentifier,
}),
ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));
return(new SentJob(JobApi, newJob));
}
public async Task <ISentJob> SendAsync(Action <string> log, CancellationToken cancellationToken)
{
IBlobHelper storage;
if (string.IsNullOrEmpty(StorageAccountConnectionString))
{
storage = new ApiBlobHelper(HelixApi.Storage);
}
else
{
storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
}
var(queueId, dockerTag, queueAlias) = ParseQueueId(TargetQueueId);
// Save time / resources by checking that the queue isn't missing before doing any potentially expensive storage operations
try
{
QueueInfo queueInfo = await HelixApi.Information.QueueInfoAsync(queueId, false, cancellationToken);
}
// 404 = this queue does not exist, or did and was removed.
catch (RestApiException ex) when(ex.Response?.Status == 404)
{
throw new ArgumentException($"Helix API does not contain an entry for {queueId}");
}
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName, queueId, cancellationToken);
var jobList = new List <JobListEntry>();
Dictionary <string, string> correlationPayloadUris =
(await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log, cancellationToken), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);
jobList = (await Task.WhenAll(
_workItems.Select(async w =>
{
var entry = await w.SendAsync(storageContainer, TargetContainerName, log, cancellationToken);
entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
return(entry);
}
))).ToList();
string jobListJson = JsonConvert.SerializeObject(jobList, Formatting.Indented);
Uri jobListUri = await storageContainer.UploadTextAsync(
jobListJson,
$"job-list-{Guid.NewGuid()}.json",
cancellationToken);
// Don't log the sas, remove the query string.
string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");
log?.Invoke($"Created job list at {jobListUriForLogging}");
cancellationToken.ThrowIfCancellationRequested();
// Only specify the ResultContainerPrefix if both repository name and source branch are available.
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH")))
{
// Container names can only be alphanumeric (plus dashes) lowercase names, with no consecutive dashes.
// Replace / with -, make all branch and repository names lowercase, remove any characters not
// allowed in container names, and replace any string of dashes with a single dash.
Regex illegalCharacters = new Regex("[^a-z0-9-]");
Regex multipleDashes = new Regex("-{2,}");
string repoName = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME");
string branchName = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");
// ResultContainerPrefix will be <Repository Name>-<BranchName>
ResultContainerPrefix = $"{repoName}-{branchName}-".Replace("/", "-").ToLower();
ResultContainerPrefix = multipleDashes.Replace(illegalCharacters.Replace(ResultContainerPrefix, ""), "-");
}
var creationRequest = new JobCreationRequest(Type, jobListUri.ToString(), queueId)
{
Properties = _properties.ToImmutableDictionary(),
Creator = Creator,
ResultContainerPrefix = ResultContainerPrefix,
DockerTag = dockerTag,
QueueAlias = queueAlias,
};
if (string.IsNullOrEmpty(Source))
{
// We only want to specify a branch if Source wasn't already provided.
// Latest Helix Job API will 400 if both Source and any of SourcePrefix, TeamProject, Repository, or Branch are set.
InitializeSourceParameters(creationRequest);
}
else
{
creationRequest.Source = Source;
}
string jobStartIdentifier = Guid.NewGuid().ToString("N");
var newJob = await JobApi.NewAsync(creationRequest, jobStartIdentifier, cancellationToken).ConfigureAwait(false);
return(new SentJob(JobApi, newJob, newJob.ResultsUri, newJob.ResultsUriRSAS));
}
public async Task <ISentJob> SendAsync(Action <string> log = null)
{
IBlobHelper storage;
if (string.IsNullOrEmpty(StorageAccountConnectionString))
{
storage = new ApiBlobHelper(HelixApi.Storage);
}
else
{
storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
}
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);
var jobList = new List <JobListEntry>();
IBlobContainer resultsStorageContainer = null;
if (!string.IsNullOrEmpty(ResultsStorageAccountConnectionString))
{
IBlobHelper resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString);
resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetResultsContainerName);
}
else if (_withDefaultResultsContainer)
{
resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName);
}
Dictionary <string, string> correlationPayloadUris =
(await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);
jobList = (await Task.WhenAll(
_workItems.Select(async w =>
{
var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
return(entry);
}
))).ToList();
string jobListJson = JsonConvert.SerializeObject(jobList);
Uri jobListUri = await storageContainer.UploadTextAsync(
jobListJson,
$"job-list-{Guid.NewGuid()}.json");
// Don't log the sas, remove the query string.
string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");
log?.Invoke($"Created job list at {jobListUriForLogging}");
string jobStartIdentifier = Guid.NewGuid().ToString("N");
JobCreationResult newJob = await HelixApi.RetryAsync(
() => JobApi.NewAsync(
new JobCreationRequest(
Source,
Type,
Build,
_properties.ToImmutableDictionary(),
jobListUri.ToString(),
TargetQueueId)
{
Creator = Creator,
MaxRetryCount = MaxRetryCount ?? 0,
JobStartIdentifier = jobStartIdentifier,
ResultsUri = resultsStorageContainer?.Uri,
ResultsUriRSAS = resultsStorageContainer?.ReadSas,
ResultsUriWSAS = resultsStorageContainer?.WriteSas,
}),
ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));
return(new SentJob(JobApi, newJob, resultsStorageContainer?.Uri, string.IsNullOrEmpty(Creator) ? resultsStorageContainer?.ReadSas : string.Empty));
}
public async Task <ISentJob> SendAsync(Action <string> log = null)
{
IBlobHelper storage;
if (string.IsNullOrEmpty(StorageAccountConnectionString))
{
storage = new ApiBlobHelper(HelixApi.Storage);
}
else
{
storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
}
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);
var jobList = new List <JobListEntry>();
IBlobContainer resultsStorageContainer = null;
if (!string.IsNullOrEmpty(ResultsStorageAccountConnectionString))
{
IBlobHelper resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString);
resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetResultsContainerName);
}
else if (_withDefaultResultsContainer)
{
resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName);
}
Dictionary <string, string> correlationPayloadUris =
(await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);
jobList = (await Task.WhenAll(
_workItems.Select(async w =>
{
var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
return(entry);
}
))).ToList();
string jobListJson = JsonConvert.SerializeObject(jobList);
Uri jobListUri = await storageContainer.UploadTextAsync(
jobListJson,
$"job-list-{Guid.NewGuid()}.json");
// Don't log the sas, remove the query string.
string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");
log?.Invoke($"Created job list at {jobListUriForLogging}");
// Only specify the ResultContainerPrefix if both repository name and source branch are available.
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH")))
{
// Container names can only be alphanumeric (plus dashes) lowercase names, with no consecutive dashes.
// Replace / with -, make all branch and repository names lowercase, remove any characters not
// allowed in container names, and replace any string of dashes with a single dash.
Regex illegalCharacters = new Regex("[^a-z0-9-]");
Regex multipleDashes = new Regex("-{2,}");
string repoName = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME");
string branchName = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");
// ResultContainerPrefix will be <Repository Name>-<BranchName>
ResultContainerPrefix = $"{repoName}-{branchName}-".Replace("/", "-").ToLower();
ResultContainerPrefix = multipleDashes.Replace(illegalCharacters.Replace(ResultContainerPrefix, ""), "-");
}
string jobStartIdentifier = Guid.NewGuid().ToString("N");
JobCreationResult newJob = await HelixApi.RetryAsync(
() => JobApi.NewAsync(
new JobCreationRequest(
Source,
Type,
Build,
_properties.ToImmutableDictionary(),
jobListUri.ToString(),
TargetQueueId)
{
Creator = Creator,
MaxRetryCount = MaxRetryCount ?? 0,
JobStartIdentifier = jobStartIdentifier,
ResultsUri = resultsStorageContainer?.Uri,
ResultsUriRSAS = resultsStorageContainer?.ReadSas,
ResultsUriWSAS = resultsStorageContainer?.WriteSas,
ResultContainerPrefix = ResultContainerPrefix,
}),
ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));
return(new SentJob(JobApi, newJob, resultsStorageContainer?.Uri, string.IsNullOrEmpty(Creator) ? resultsStorageContainer?.ReadSas : string.Empty));
}
public async Task <ISentJob> SendAsync(Action <string> log = null)
{
IBlobHelper storage;
if (string.IsNullOrEmpty(StorageAccountConnectionString))
{
storage = new ApiBlobHelper(HelixApi.Storage);
}
else
{
storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
}
var(queueId, dockerTag, queueAlias) = ParseQueueId(TargetQueueId);
IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName, queueId);
var jobList = new List <JobListEntry>();
Dictionary <string, string> correlationPayloadUris =
(await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);
jobList = (await Task.WhenAll(
_workItems.Select(async w =>
{
var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
return(entry);
}
))).ToList();
string jobListJson = JsonConvert.SerializeObject(jobList);
Uri jobListUri = await storageContainer.UploadTextAsync(
jobListJson,
$"job-list-{Guid.NewGuid()}.json");
// Don't log the sas, remove the query string.
string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");
log?.Invoke($"Created job list at {jobListUriForLogging}");
// Only specify the ResultContainerPrefix if both repository name and source branch are available.
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH")))
{
// Container names can only be alphanumeric (plus dashes) lowercase names, with no consecutive dashes.
// Replace / with -, make all branch and repository names lowercase, remove any characters not
// allowed in container names, and replace any string of dashes with a single dash.
Regex illegalCharacters = new Regex("[^a-z0-9-]");
Regex multipleDashes = new Regex("-{2,}");
string repoName = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME");
string branchName = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");
// ResultContainerPrefix will be <Repository Name>-<BranchName>
ResultContainerPrefix = $"{repoName}-{branchName}-".Replace("/", "-").ToLower();
ResultContainerPrefix = multipleDashes.Replace(illegalCharacters.Replace(ResultContainerPrefix, ""), "-");
}
var creationRequest = new JobCreationRequest(Type, _properties.ToImmutableDictionary(), jobListUri.ToString(), queueId)
{
Creator = Creator,
ResultContainerPrefix = ResultContainerPrefix,
DockerTag = dockerTag,
QueueAlias = queueAlias,
};
if (string.IsNullOrEmpty(Source))
{
// We only want to specify a branch if Source wasn't already provided.
// Latest Helix Job API will 400 if both Source and any of SourcePrefix, TeamProject, Repository, or Branch are set.
InitializeSourceParameters(creationRequest);
}
else
{
creationRequest.Source = Source;
}
string jobStartIdentifier = Guid.NewGuid().ToString("N");
JobCreationResult newJob = await HelixApi.RetryAsync(
() => JobApi.NewAsync(creationRequest, jobStartIdentifier),
ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."),
IsRetryableJobListUriHttpError,
CancellationToken.None);
return(new SentJob(JobApi, newJob, newJob.ResultsUri, newJob.ResultsUriRSAS));
}