Example #1
0
        public async Task <ISentJob> SendAsync(Action <string> log = null)
        {
            IBlobHelper storage;

            if (string.IsNullOrEmpty(StorageAccountConnectionString))
            {
                storage = new ApiBlobHelper(HelixApi.Storage);
            }
            else
            {
                storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
            }

            IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);

            var jobList = new List <JobListEntry>();

            List <string> correlationPayloadUris =
                (await Task.WhenAll(CorrelationPayloads.Select(p => p.UploadAsync(storageContainer, log)))).ToList();

            jobList = (await Task.WhenAll(
                           _workItems.Select(async w =>
            {
                var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
                entry.CorrelationPayloadUris = correlationPayloadUris;
                return(entry);
            }
                                             ))).ToList();

            string jobListJson = JsonConvert.SerializeObject(jobList);
            Uri    jobListUri  = await storageContainer.UploadTextAsync(
                jobListJson,
                $"job-list-{Guid.NewGuid()}.json");


            string            jobStartIdentifier = Guid.NewGuid().ToString("N");
            JobCreationResult newJob             = await HelixApi.RetryAsync(
                () => JobApi.NewAsync(
                    new JobCreationRequest(
                        Source,
                        Type,
                        Build,
                        _properties.ToImmutableDictionary(),
                        jobListUri.ToString(),
                        TargetQueueId,
                        storageContainer.Uri,
                        storageContainer.ReadSas,
                        storageContainer.WriteSas)
            {
                Creator            = Creator,
                MaxRetryCount      = MaxRetryCount ?? 0,
                JobStartIdentifier = jobStartIdentifier,
            }),
                ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));


            return(new SentJob(JobApi, newJob));
        }
Example #2
0
        public async Task <ISentJob> SendAsync(Action <string> log, CancellationToken cancellationToken)
        {
            IBlobHelper storage;

            if (string.IsNullOrEmpty(StorageAccountConnectionString))
            {
                storage = new ApiBlobHelper(HelixApi.Storage);
            }
            else
            {
                storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
            }

            var(queueId, dockerTag, queueAlias) = ParseQueueId(TargetQueueId);

            // Save time / resources by checking that the queue isn't missing before doing any potentially expensive storage operations
            try
            {
                QueueInfo queueInfo = await HelixApi.Information.QueueInfoAsync(queueId, false, cancellationToken);
            }
            // 404 = this queue does not exist, or did and was removed.
            catch (RestApiException ex) when(ex.Response?.Status == 404)
            {
                throw new ArgumentException($"Helix API does not contain an entry for {queueId}");
            }

            IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName, queueId, cancellationToken);

            var jobList = new List <JobListEntry>();

            Dictionary <string, string> correlationPayloadUris =
                (await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log, cancellationToken), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);

            jobList = (await Task.WhenAll(
                           _workItems.Select(async w =>
            {
                var entry = await w.SendAsync(storageContainer, TargetContainerName, log, cancellationToken);
                entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
                return(entry);
            }
                                             ))).ToList();

            string jobListJson = JsonConvert.SerializeObject(jobList, Formatting.Indented);
            Uri    jobListUri  = await storageContainer.UploadTextAsync(
                jobListJson,
                $"job-list-{Guid.NewGuid()}.json",
                cancellationToken);

            // Don't log the sas, remove the query string.
            string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");

            log?.Invoke($"Created job list at {jobListUriForLogging}");

            cancellationToken.ThrowIfCancellationRequested();

            // Only specify the ResultContainerPrefix if both repository name and source branch are available.
            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH")))
            {
                // Container names can only be alphanumeric (plus dashes) lowercase names, with no consecutive dashes.
                // Replace / with -, make all branch and repository names lowercase, remove any characters not
                // allowed in container names, and replace any string of dashes with a single dash.
                Regex illegalCharacters = new Regex("[^a-z0-9-]");
                Regex multipleDashes    = new Regex("-{2,}");

                string repoName   = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME");
                string branchName = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");

                // ResultContainerPrefix will be <Repository Name>-<BranchName>
                ResultContainerPrefix = $"{repoName}-{branchName}-".Replace("/", "-").ToLower();
                ResultContainerPrefix = multipleDashes.Replace(illegalCharacters.Replace(ResultContainerPrefix, ""), "-");
            }

            var creationRequest = new JobCreationRequest(Type, jobListUri.ToString(), queueId)
            {
                Properties            = _properties.ToImmutableDictionary(),
                Creator               = Creator,
                ResultContainerPrefix = ResultContainerPrefix,
                DockerTag             = dockerTag,
                QueueAlias            = queueAlias,
            };

            if (string.IsNullOrEmpty(Source))
            {
                // We only want to specify a branch if Source wasn't already provided.
                // Latest Helix Job API will 400 if both Source and any of SourcePrefix, TeamProject, Repository, or Branch are set.
                InitializeSourceParameters(creationRequest);
            }
            else
            {
                creationRequest.Source = Source;
            }

            string jobStartIdentifier = Guid.NewGuid().ToString("N");
            var    newJob             = await JobApi.NewAsync(creationRequest, jobStartIdentifier, cancellationToken).ConfigureAwait(false);

            return(new SentJob(JobApi, newJob, newJob.ResultsUri, newJob.ResultsUriRSAS));
        }
Example #3
0
        public async Task <ISentJob> SendAsync(Action <string> log = null)
        {
            IBlobHelper storage;

            if (string.IsNullOrEmpty(StorageAccountConnectionString))
            {
                storage = new ApiBlobHelper(HelixApi.Storage);
            }
            else
            {
                storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
            }

            IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);

            var jobList = new List <JobListEntry>();

            IBlobContainer resultsStorageContainer = null;

            if (!string.IsNullOrEmpty(ResultsStorageAccountConnectionString))
            {
                IBlobHelper resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString);
                resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetResultsContainerName);
            }
            else if (_withDefaultResultsContainer)
            {
                resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName);
            }

            Dictionary <string, string> correlationPayloadUris =
                (await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);

            jobList = (await Task.WhenAll(
                           _workItems.Select(async w =>
            {
                var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
                entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
                return(entry);
            }
                                             ))).ToList();

            string jobListJson = JsonConvert.SerializeObject(jobList);
            Uri    jobListUri  = await storageContainer.UploadTextAsync(
                jobListJson,
                $"job-list-{Guid.NewGuid()}.json");

            // Don't log the sas, remove the query string.
            string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");

            log?.Invoke($"Created job list at {jobListUriForLogging}");

            string            jobStartIdentifier = Guid.NewGuid().ToString("N");
            JobCreationResult newJob             = await HelixApi.RetryAsync(
                () => JobApi.NewAsync(
                    new JobCreationRequest(
                        Source,
                        Type,
                        Build,
                        _properties.ToImmutableDictionary(),
                        jobListUri.ToString(),
                        TargetQueueId)
            {
                Creator            = Creator,
                MaxRetryCount      = MaxRetryCount ?? 0,
                JobStartIdentifier = jobStartIdentifier,
                ResultsUri         = resultsStorageContainer?.Uri,
                ResultsUriRSAS     = resultsStorageContainer?.ReadSas,
                ResultsUriWSAS     = resultsStorageContainer?.WriteSas,
            }),
                ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));


            return(new SentJob(JobApi, newJob, resultsStorageContainer?.Uri, string.IsNullOrEmpty(Creator) ? resultsStorageContainer?.ReadSas : string.Empty));
        }
Example #4
0
        public async Task <ISentJob> SendAsync(Action <string> log = null)
        {
            IBlobHelper storage;

            if (string.IsNullOrEmpty(StorageAccountConnectionString))
            {
                storage = new ApiBlobHelper(HelixApi.Storage);
            }
            else
            {
                storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
            }

            IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName);

            var jobList = new List <JobListEntry>();

            IBlobContainer resultsStorageContainer = null;

            if (!string.IsNullOrEmpty(ResultsStorageAccountConnectionString))
            {
                IBlobHelper resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString);
                resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetResultsContainerName);
            }
            else if (_withDefaultResultsContainer)
            {
                resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName);
            }

            Dictionary <string, string> correlationPayloadUris =
                (await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);

            jobList = (await Task.WhenAll(
                           _workItems.Select(async w =>
            {
                var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
                entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
                return(entry);
            }
                                             ))).ToList();

            string jobListJson = JsonConvert.SerializeObject(jobList);
            Uri    jobListUri  = await storageContainer.UploadTextAsync(
                jobListJson,
                $"job-list-{Guid.NewGuid()}.json");

            // Don't log the sas, remove the query string.
            string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");

            log?.Invoke($"Created job list at {jobListUriForLogging}");

            // Only specify the ResultContainerPrefix if both repository name and source branch are available.
            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH")))
            {
                // Container names can only be alphanumeric (plus dashes) lowercase names, with no consecutive dashes.
                // Replace / with -, make all branch and repository names lowercase, remove any characters not
                // allowed in container names, and replace any string of dashes with a single dash.
                Regex illegalCharacters = new Regex("[^a-z0-9-]");
                Regex multipleDashes    = new Regex("-{2,}");

                string repoName   = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME");
                string branchName = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");

                // ResultContainerPrefix will be <Repository Name>-<BranchName>
                ResultContainerPrefix = $"{repoName}-{branchName}-".Replace("/", "-").ToLower();
                ResultContainerPrefix = multipleDashes.Replace(illegalCharacters.Replace(ResultContainerPrefix, ""), "-");
            }

            string            jobStartIdentifier = Guid.NewGuid().ToString("N");
            JobCreationResult newJob             = await HelixApi.RetryAsync(
                () => JobApi.NewAsync(
                    new JobCreationRequest(
                        Source,
                        Type,
                        Build,
                        _properties.ToImmutableDictionary(),
                        jobListUri.ToString(),
                        TargetQueueId)
            {
                Creator               = Creator,
                MaxRetryCount         = MaxRetryCount ?? 0,
                JobStartIdentifier    = jobStartIdentifier,
                ResultsUri            = resultsStorageContainer?.Uri,
                ResultsUriRSAS        = resultsStorageContainer?.ReadSas,
                ResultsUriWSAS        = resultsStorageContainer?.WriteSas,
                ResultContainerPrefix = ResultContainerPrefix,
            }),
                ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."));


            return(new SentJob(JobApi, newJob, resultsStorageContainer?.Uri, string.IsNullOrEmpty(Creator) ? resultsStorageContainer?.ReadSas : string.Empty));
        }
Example #5
0
        public async Task <ISentJob> SendAsync(Action <string> log = null)
        {
            IBlobHelper storage;

            if (string.IsNullOrEmpty(StorageAccountConnectionString))
            {
                storage = new ApiBlobHelper(HelixApi.Storage);
            }
            else
            {
                storage = new ConnectionStringBlobHelper(StorageAccountConnectionString);
            }

            var(queueId, dockerTag, queueAlias) = ParseQueueId(TargetQueueId);

            IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName, queueId);

            var jobList = new List <JobListEntry>();

            Dictionary <string, string> correlationPayloadUris =
                (await Task.WhenAll(CorrelationPayloads.Select(async p => (uri: await p.Key.UploadAsync(storageContainer, log), destination: p.Value)))).ToDictionary(x => x.uri, x => x.destination);

            jobList = (await Task.WhenAll(
                           _workItems.Select(async w =>
            {
                var entry = await w.SendAsync(storageContainer, TargetContainerName, log);
                entry.CorrelationPayloadUrisWithDestinations = correlationPayloadUris;
                return(entry);
            }
                                             ))).ToList();

            string jobListJson = JsonConvert.SerializeObject(jobList);
            Uri    jobListUri  = await storageContainer.UploadTextAsync(
                jobListJson,
                $"job-list-{Guid.NewGuid()}.json");

            // Don't log the sas, remove the query string.
            string jobListUriForLogging = jobListUri.ToString().Replace(jobListUri.Query, "");

            log?.Invoke($"Created job list at {jobListUriForLogging}");

            // Only specify the ResultContainerPrefix if both repository name and source branch are available.
            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH")))
            {
                // Container names can only be alphanumeric (plus dashes) lowercase names, with no consecutive dashes.
                // Replace / with -, make all branch and repository names lowercase, remove any characters not
                // allowed in container names, and replace any string of dashes with a single dash.
                Regex illegalCharacters = new Regex("[^a-z0-9-]");
                Regex multipleDashes    = new Regex("-{2,}");

                string repoName   = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME");
                string branchName = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");

                // ResultContainerPrefix will be <Repository Name>-<BranchName>
                ResultContainerPrefix = $"{repoName}-{branchName}-".Replace("/", "-").ToLower();
                ResultContainerPrefix = multipleDashes.Replace(illegalCharacters.Replace(ResultContainerPrefix, ""), "-");
            }

            var creationRequest = new JobCreationRequest(Type, _properties.ToImmutableDictionary(), jobListUri.ToString(), queueId)
            {
                Creator = Creator,
                ResultContainerPrefix = ResultContainerPrefix,
                DockerTag             = dockerTag,
                QueueAlias            = queueAlias,
            };

            if (string.IsNullOrEmpty(Source))
            {
                // We only want to specify a branch if Source wasn't already provided.
                // Latest Helix Job API will 400 if both Source and any of SourcePrefix, TeamProject, Repository, or Branch are set.
                InitializeSourceParameters(creationRequest);
            }
            else
            {
                creationRequest.Source = Source;
            }

            string            jobStartIdentifier = Guid.NewGuid().ToString("N");
            JobCreationResult newJob             = await HelixApi.RetryAsync(
                () => JobApi.NewAsync(creationRequest, jobStartIdentifier),
                ex => log?.Invoke($"Starting job failed with {ex}\nRetrying..."),
                IsRetryableJobListUriHttpError,
                CancellationToken.None);

            return(new SentJob(JobApi, newJob, newJob.ResultsUri, newJob.ResultsUriRSAS));
        }