public async Task <string> SendJob() { Console.WriteLine($"Sending job to Helix..."); var sentJob = await _job.SendAsync(); Console.WriteLine($"Job '{sentJob.CorrelationId}' created."); return(sentJob.CorrelationId); }
protected override async Task ExecuteCore() { using (_commandPayload = new CommandPayload(this)) { IJobDefinition def = HelixApi.Job.Define() .WithSource(Source) .WithType(Type) .WithBuild(Build) .WithTargetQueue(TargetQueue); if (CorrelationPayloads != null) { foreach (ITaskItem correlationPayload in CorrelationPayloads) { def = AddCorrelationPayload(def, correlationPayload); } } if (WorkItems != null) { foreach (ITaskItem workItem in WorkItems) { def = AddWorkItem(def, workItem); } } else { Log.LogError("SendHelixJob given no WorkItems to send."); } if (_commandPayload.TryGetPayloadDirectory(out string directory)) { def = def.WithCorrelationPayloadDirectory(directory); } // don't send the job if we have errors if (Log.HasLoggedErrors) { return; } Log.LogMessage(MessageImportance.Normal, "Sending Job..."); ISentJob job = await def.SendAsync(); JobCorrelationId = job.CorrelationId; } }
protected override async Task ExecuteCore() { IJobDefinition def = HelixApi.Job.Define() .WithSource(Source) .WithType(Type) .WithBuild(Build) .WithTargetQueue(TargetQueue); if (CorrelationPayloads != null) { foreach (ITaskItem correlationPayload in CorrelationPayloads) { def = AddCorrelationPayload(def, correlationPayload); } } if (WorkItems != null) { foreach (ITaskItem workItem in WorkItems) { def = AddWorkItem(def, workItem); } } else { Log.LogError("SendHelixJob given no WorkItems to send."); } // don't send the job if we have errors if (Log.HasLoggedErrors) { return; } ISentJob job = await def.SendAsync(); JobCorrelationId = job.CorrelationId; }
protected override async Task ExecuteCore(CancellationToken cancellationToken) { if (string.IsNullOrEmpty(AccessToken) && string.IsNullOrEmpty(Creator)) { Log.LogError(FailureCategory.Build, "Creator is required when using anonymous access."); return; } if (!string.IsNullOrEmpty(AccessToken) && !string.IsNullOrEmpty(Creator)) { Log.LogError(FailureCategory.Build, "Creator is forbidden when using authenticated access."); return; } Type = Type.ToLowerInvariant(); cancellationToken.ThrowIfCancellationRequested(); using (_commandPayload = new CommandPayload(this)) { var currentHelixApi = HelixApi; IJobDefinition def = currentHelixApi.Job.Define() .WithType(Type) .WithTargetQueue(TargetQueue) .WithMaxRetryCount(MaxRetryCount); Log.LogMessage($"Initialized job definition with type '{Type}', and target queue '{TargetQueue}'"); if (!string.IsNullOrEmpty(Creator)) { def = def.WithCreator(Creator); Log.LogMessage($"Setting creator to '{Creator}'"); } Log.LogMessage(MessageImportance.High, $"Uploading payloads for Job on {TargetQueue}..."); if (CorrelationPayloads != null) { foreach (ITaskItem correlationPayload in CorrelationPayloads) { def = AddCorrelationPayload(def, correlationPayload); } } if (WorkItems != null) { foreach (ITaskItem workItem in WorkItems) { def = AddWorkItem(def, workItem); } } else { Log.LogError(FailureCategory.Build, "SendHelixJob given no WorkItems to send."); } if (_commandPayload.TryGetPayloadDirectory(out string directory)) { def = def.WithCorrelationPayloadDirectory(directory); } Log.LogMessage(MessageImportance.High, $"Finished uploading payloads for Job on {TargetQueue}..."); if (HelixProperties != null) { foreach (ITaskItem helixProperty in HelixProperties) { def = AddProperty(def, helixProperty); } } // don't send the job if we have errors if (Log.HasLoggedErrors) { return; } Log.LogMessage(MessageImportance.High, $"Sending Job to {TargetQueue}..."); cancellationToken.ThrowIfCancellationRequested(); ISentJob job = await def.SendAsync(msg => Log.LogMessage(msg), cancellationToken); JobCorrelationId = job.CorrelationId; ResultsContainerUri = job.ResultsContainerUri; ResultsContainerReadSAS = job.ResultsContainerReadSAS; cancellationToken.ThrowIfCancellationRequested(); } cancellationToken.ThrowIfCancellationRequested(); }
protected override async Task ExecuteCore(CancellationToken cancellationToken) { if (string.IsNullOrEmpty(AccessToken) && string.IsNullOrEmpty(Creator)) { Log.LogError("Creator is required when using anonymous access."); return; } if (!string.IsNullOrEmpty(AccessToken) && !string.IsNullOrEmpty(Creator)) { Log.LogError("Creator is forbidden when using authenticated access."); return; } Source = Source.ToLowerInvariant(); Type = Type.ToLowerInvariant(); Build = Build.ToLowerInvariant(); cancellationToken.ThrowIfCancellationRequested(); using (_commandPayload = new CommandPayload(this)) { var currentHelixApi = HelixApi; IJobDefinition def = currentHelixApi.Job.Define() .WithSource(Source) .WithType(Type) .WithBuild(Build) .WithTargetQueue(TargetQueue) .WithMaxRetryCount(MaxRetryCount); Log.LogMessage($"Initialized job definition with source '{Source}', type '{Type}', build number '{Build}', and target queue '{TargetQueue}'"); if (!string.IsNullOrEmpty(Creator)) { def = def.WithCreator(Creator); Log.LogMessage($"Setting creator to '{Creator}'"); } if (CorrelationPayloads != null) { foreach (ITaskItem correlationPayload in CorrelationPayloads) { def = AddCorrelationPayload(def, correlationPayload); } } if (WorkItems != null) { foreach (ITaskItem workItem in WorkItems) { def = AddWorkItem(def, workItem); } } else { Log.LogError("SendHelixJob given no WorkItems to send."); } if (_commandPayload.TryGetPayloadDirectory(out string directory)) { def = def.WithCorrelationPayloadDirectory(directory); } if (HelixProperties != null) { foreach (ITaskItem helixProperty in HelixProperties) { def = AddProperty(def, helixProperty); } } if (UsingDownloadResultsFeature) { def = def.WithDefaultResultsContainer(); } // don't send the job if we have errors if (Log.HasLoggedErrors) { return; } Log.LogMessage(MessageImportance.Normal, "Sending Job..."); cancellationToken.ThrowIfCancellationRequested(); ISentJob job = await def.SendAsync(msg => Log.LogMessage(msg)); JobCorrelationId = job.CorrelationId; ResultsContainerUri = job.ResultsContainerUri; ResultsContainerReadSAS = job.ResultsContainerReadSAS; cancellationToken.ThrowIfCancellationRequested(); } string mcUri = await GetMissionControlResultUri(); Log.LogMessage(MessageImportance.High, $"Results will be available from {mcUri}"); cancellationToken.ThrowIfCancellationRequested(); }
protected override async Task ExecuteCore() { if (string.IsNullOrEmpty(AccessToken) && string.IsNullOrEmpty(Creator)) { Log.LogError("Creator is required when using anonymous access."); return; } if (!string.IsNullOrEmpty(AccessToken) && !string.IsNullOrEmpty(Creator)) { Log.LogError("Creator is forbidden when using authenticated access."); return; } using (_commandPayload = new CommandPayload(this)) { var currentHelixApi = HelixApi; IJobDefinition def = currentHelixApi.Job.Define() .WithSource(Source) .WithType(Type) .WithBuild(Build) .WithTargetQueue(TargetQueue) .WithMaxRetryCount(MaxRetryCount); Log.LogMessage($"Initialized job definition with source '{Source}', type '{Type}', build number '{Build}', and target queue '{TargetQueue}'"); if (!string.IsNullOrEmpty(Creator)) { def = def.WithCreator(Creator); Log.LogMessage($"Setting creator to '{Creator}'"); } if (CorrelationPayloads != null) { foreach (ITaskItem correlationPayload in CorrelationPayloads) { def = AddCorrelationPayload(def, correlationPayload); } } if (WorkItems != null) { foreach (ITaskItem workItem in WorkItems) { def = AddWorkItem(def, workItem); } } else { Log.LogError("SendHelixJob given no WorkItems to send."); } if (_commandPayload.TryGetPayloadDirectory(out string directory)) { def = def.WithCorrelationPayloadDirectory(directory); } if (HelixProperties != null) { foreach (ITaskItem helixProperty in HelixProperties) { def = AddProperty(def, helixProperty); } } // don't send the job if we have errors if (Log.HasLoggedErrors) { return; } Log.LogMessage(MessageImportance.Normal, "Sending Job..."); ISentJob job = await def.SendAsync(msg => Log.LogMessage(msg)); JobCorrelationId = job.CorrelationId; } }
protected override async Task ExecuteCore(CancellationToken cancellationToken) { if (string.IsNullOrEmpty(AccessToken) && string.IsNullOrEmpty(Creator)) { Log.LogError(FailureCategory.Build, "Creator is required when using anonymous access."); return; } if (!string.IsNullOrEmpty(AccessToken) && !string.IsNullOrEmpty(Creator)) { Log.LogError(FailureCategory.Build, "Creator is forbidden when using authenticated access."); return; } Type = Type.ToLowerInvariant(); cancellationToken.ThrowIfCancellationRequested(); using (_commandPayload = new CommandPayload(this)) { var currentHelixApi = HelixApi; IJobDefinition def = currentHelixApi.Job.Define() .WithType(Type) .WithTargetQueue(TargetQueue) .WithMaxRetryCount(MaxRetryCount); Log.LogMessage($"Initialized job definition with type '{Type}', and target queue '{TargetQueue}'"); if (!string.IsNullOrEmpty(Creator)) { def = def.WithCreator(Creator); Log.LogMessage($"Setting creator to '{Creator}'"); } if (CorrelationPayloads == null) { Log.LogMessage($"No Correlation Payloads for Job on {TargetQueue} set"); } else { Log.LogMessage($"Adding Correlation Payloads for Job on {TargetQueue}..."); foreach (ITaskItem correlationPayload in CorrelationPayloads) { def = AddCorrelationPayload(def, correlationPayload); } } if (WorkItems != null) { foreach (ITaskItem workItem in WorkItems) { def = AddWorkItem(def, workItem); } } else { Log.LogError(FailureCategory.Build, "SendHelixJob given no WorkItems to send."); } if (_commandPayload.TryGetPayloadDirectory(out string directory)) { def = def.WithCorrelationPayloadDirectory(directory); } if (HelixProperties != null) { foreach (ITaskItem helixProperty in HelixProperties) { def = AddProperty(def, helixProperty); } } def = AddBuildVariableProperty(def, "CollectionUri", "System.CollectionUri"); def = AddBuildVariableProperty(def, "Project", "System.TeamProject"); def = AddBuildVariableProperty(def, "BuildNumber", "Build.BuildNumber"); def = AddBuildVariableProperty(def, "BuildId", "Build.BuildId"); def = AddBuildVariableProperty(def, "DefinitionName", "Build.DefinitionName"); def = AddBuildVariableProperty(def, "DefinitionId", "System.DefinitionId"); def = AddBuildVariableProperty(def, "Reason", "Build.Reason"); var variablesToCopy = new[] { "System.JobId", "System.JobName", "System.JobAttempt", "System.PhaseName", "System.PhaseAttempt", "System.PullRequest.TargetBranch", "System.StageName", "System.StageAttempt", }; foreach (var name in variablesToCopy) { def = AddBuildVariableProperty(def, name, name); } // don't send the job if we have errors if (Log.HasLoggedErrors) { return; } Log.LogMessage(MessageImportance.High, $"Sending Job to {TargetQueue}..."); cancellationToken.ThrowIfCancellationRequested(); // LogMessageFromText will take any string formatted as a canonical error or warning and convert the type of log to this ISentJob job = await def.SendAsync(msg => Log.LogMessageFromText(msg, MessageImportance.Normal), cancellationToken); JobCorrelationId = job.CorrelationId; JobCancellationToken = job.HelixCancellationToken; ResultsContainerUri = job.ResultsContainerUri; ResultsContainerReadSAS = job.ResultsContainerReadSAS; cancellationToken.ThrowIfCancellationRequested(); } cancellationToken.ThrowIfCancellationRequested(); }
protected override async Task ExecuteCore() { using (_commandPayload = new CommandPayload(this)) { var currentHelixApi = HelixApi; if (IsExternal) { Log.LogMessage($"Job is external. Switching to anonymous API."); currentHelixApi = AnonymousApi; var storageApi = new Storage((HelixApi)HelixApi); typeof(HelixApi).GetProperty("Storage").SetValue(AnonymousApi, storageApi); } IJobDefinition def = currentHelixApi.Job.Define() .WithSource(Source) .WithType(Type) .WithBuild(Build) .WithTargetQueue(TargetQueue) .WithMaxRetryCount(MaxRetryCount); Log.LogMessage($"Initialized job definition with source '{Source}', type '{Type}', build number '{Build}', and target queue '{TargetQueue}'"); if (IsExternal) { if (string.IsNullOrEmpty(Creator)) { Log.LogError("The Creator property was left unspecified for an external job. Please set the Creator property or set IsExternal to false."); } else { def.WithCreator(Creator); Log.LogMessage($"Setting creator to '{Creator}'"); } } if (CorrelationPayloads != null) { foreach (ITaskItem correlationPayload in CorrelationPayloads) { def = AddCorrelationPayload(def, correlationPayload); } } if (WorkItems != null) { foreach (ITaskItem workItem in WorkItems) { def = AddWorkItem(def, workItem); } } else { Log.LogError("SendHelixJob given no WorkItems to send."); } if (_commandPayload.TryGetPayloadDirectory(out string directory)) { def = def.WithCorrelationPayloadDirectory(directory); } if (HelixProperties != null) { foreach (ITaskItem helixProperty in HelixProperties) { def = AddProperty(def, helixProperty); } } // don't send the job if we have errors if (Log.HasLoggedErrors) { return; } Log.LogMessage(MessageImportance.Normal, "Sending Job..."); ISentJob job = await def.SendAsync(msg => Log.LogMessage(msg)); JobCorrelationId = job.CorrelationId; } }
public async Task <AgentInfoItem> CreateJob( CancellationToken cancellationToken, Dictionary <string, string> properties) { string credentialsPath = null; string agentSettingsPath = null; ISentJob sentJob = null; try { _logger.LogInformation($"Creating payloads for agent id {_agentRequestItem.agentId}"); credentialsPath = CreateAgentCredentialsPayload(); agentSettingsPath = CreateAgentSettingsPayload(); // Now that we have a valid queue, construct the Helix job on that queue // Notes: if we timeout, it's going to be in the subsequent call. // SendAsync() causes both the storage account container creation / uploads and sends to Helix API, which both can stall. // We have to do this this way (and non-ideal workarounds like trying to destroy the object won't likely solve this) because today // the job, if started, will still contain valid Azure DevOps tokens to be a build agent and we can't easily guarantee it doesn't get sent. // ******************************** // The right long-term fix is for Azure DevOps to not have the tokens passed be usable until they've received an "accepted = true" response from our provider. // ******************************** cancellationToken.ThrowIfCancellationRequested(); IJobDefinition preparedJob = _api.Job.Define() .WithType($"byoc/{_configuration.HelixCreator}/") .WithTargetQueue(_queueInfo.QueueId); if (properties != null) { foreach ((string key, string value) in properties) { preparedJob.WithProperty(key, value); } } preparedJob = preparedJob.WithContainerName(_configuration.ContainerName) .WithCorrelationPayloadUris(AgentPayloadUri) .WithSource($"agent/{_agentRequestItem.accountId}/{_orchestrationId}/{_jobName}/"); IWorkItemDefinition workitem = preparedJob.DefineWorkItem(_agentRequestItem.agentId) .WithCommand(ConstructCommand()) .WithFiles(credentialsPath, agentSettingsPath, StartupScriptPath) .WithTimeout(TimeSpan.FromMinutes(_configuration.TimeoutInMinutes)); preparedJob = workitem.AttachToJob(); sentJob = await preparedJob.SendAsync(l => _logger.LogInformation(l), cancellationToken); _logger.LogInformation($"Successfully submitted new Helix job {sentJob.CorrelationId} (Agent id {_agentRequestItem.agentId}) to queue { _queueInfo.QueueId}"); // In case the cancellation token got signalled between above and here, let's try to cancel the Helix Job. cancellationToken.ThrowIfCancellationRequested(); return(new AgentInfoItem { accepted = true, agentData = new AgentDataItem { correlationId = sentJob.CorrelationId, queueId = _queueInfo.QueueId, workItemId = _agentRequestItem.agentId, isPublicQueue = !_queueInfo.IsInternalOnly.GetValueOrDefault(true) } }); } catch (HttpOperationException e) { _logger.LogError(e, $"Failed to submit new Helix job to queue {_queueInfo.QueueId} for agent id {_agentRequestItem.agentId}: {e.Response.Content}"); return(new AgentInfoItem() { accepted = false }); } catch (OperationCanceledException ranOutOfTime) when(ranOutOfTime.CancellationToken == cancellationToken) { _logger.LogError($"Unable to complete request to create Helix job in specified timeout, attempting to cancel it."); if (sentJob != null && !string.IsNullOrEmpty(sentJob.CorrelationId)) { await _api.Job.CancelAsync(sentJob.CorrelationId); _logger.LogError($"Possible race condition: cancelled Helix Job '{sentJob.CorrelationId}' may still run."); } return(new AgentInfoItem() { accepted = false }); } catch (Exception e) { _logger.LogError(e, $"Failed to submit new Helix job to queue {_queueInfo.QueueId} for agent id {_agentRequestItem.agentId}"); return(new AgentInfoItem() { accepted = false }); } finally { if (credentialsPath != null) { // Delete the temporary files containing the credentials and agent config System.IO.File.Delete(credentialsPath); } if (agentSettingsPath != null) { System.IO.File.Delete(agentSettingsPath); } } }