public static void HandleSocketException(SocketException e, string url, Action <string> traceErrorAction) { traceErrorAction("SocketException occurred."); traceErrorAction(e.Message); traceErrorAction($"Verify whether you have (network) access to { url }"); traceErrorAction($"URLs the agent need communicate with - { BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform() }"); }
private async Task DownloadFileContainerAsync(IEnumerable <FileContainerItem> items, ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); // Only initialize these clients if we know we need to download from Blobstore // If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean(); DedupStoreClient dedupClient = null; BlobStoreClientTelemetryTfs clientTelemetry = null; if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null)) { try { (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync( false, (str) => this.tracer.Info(str), this.connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken); } catch (SocketException e) { ExceptionsUtil.HandleSocketException(e, connection.Uri.ToString(), context.Warning); } catch { var blobStoreHost = dedupClient.Client.BaseAddress.Host; var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform(); var warningMessage = StringUtil.Loc("BlobStoreDownloadWarning", blobStoreHost, allowListLink); // Fall back to streaming through TFS if we cannot reach blobstore downloadFromBlob = false; tracer.Warn(warningMessage); } } var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { tracer.Info($"Downloading: {targetPath}"); if (item.BlobMetadata != null && downloadFromBlob) { await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken); } else { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: downloadParameters.RetryDownloadCount, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); // Send results to CustomerIntelligence if (clientTelemetry != null) { var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString()); var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString()); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload, properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId)); } // check files (will throw an exception if a file is corrupt) if (downloadParameters.CheckDownloadedFiles) { CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); } }
public async Task DownloadCommitsAsync(IExecutionContext context, ArtifactDefinition artifactDefinition, string commitsWorkFolder) { ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); Trace.Entering(); var jenkinsDetails = artifactDefinition.Details as JenkinsArtifactDetails; int startJobId = 0, endJobId = 0; if (!string.IsNullOrEmpty(jenkinsDetails.EndCommitArtifactVersion)) { if (int.TryParse(jenkinsDetails.EndCommitArtifactVersion, out endJobId)) { context.Output(StringUtil.Loc("RMDownloadingCommits")); if (int.TryParse(jenkinsDetails.StartCommitArtifactVersion, out startJobId)) { if (startJobId < endJobId) { context.Output(StringUtil.Loc("DownloadingJenkinsCommitsBetween", startJobId, endJobId)); } else if (startJobId > endJobId) { context.Output(StringUtil.Loc("JenkinsRollbackDeployment", startJobId, endJobId)); // swap the job IDs to fetch the roll back commits int swap = startJobId; startJobId = endJobId; endJobId = swap; } else if (startJobId == endJobId) { context.Output(StringUtil.Loc("JenkinsNoCommitsToFetch")); return; } } else { context.Debug(StringUtil.Loc("JenkinsDownloadingChangeFromCurrentBuild")); } try { IEnumerable <Change> changes = await DownloadCommits(context, jenkinsDetails, startJobId, endJobId); if (changes.Any()) { string commitsFileName = GetCommitsFileName(jenkinsDetails.Alias); string commitsFilePath = Path.Combine(commitsWorkFolder, commitsFileName); context.Debug($"Commits will be written to {commitsFilePath}"); WriteCommitsToFile(context, changes, commitsFilePath); context.Debug($"Commits written to {commitsFilePath}"); context.QueueAttachFile(CoreAttachmentType.FileAttachment, commitsFileName, commitsFilePath); } } catch (SocketException ex) { context.AddIssue(new Issue { Type = IssueType.Error, Message = $"SocketException occurred. { ex.Message }." + $"Verify whether you have (network) access to { jenkinsDetails.Url }. URLs the agent need communicate with - { BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform() }." }); return; } catch (Exception ex) { context.AddIssue(new Issue { Type = IssueType.Warning, Message = StringUtil.Loc("DownloadingJenkinsCommitsFailedWithException", jenkinsDetails.Alias, ex.ToString()) }); return; } } else { context.AddIssue(new Issue { Type = IssueType.Warning, Message = StringUtil.Loc("JenkinsCommitsInvalidEndJobId", jenkinsDetails.EndCommitArtifactVersion, jenkinsDetails.Alias) }); return; } } else { context.Debug("No commit details found in the agent artifact. Not downloading the commits"); } }
private async Task <UploadResult> BlobUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token) { // return files that fail to upload and total artifact size var uploadResult = new UploadResult(); // nothing needs to upload if (files.Count == 0) { return(uploadResult); } DedupStoreClient dedupClient = null; BlobStoreClientTelemetryTfs clientTelemetry = null; try { var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase); int maxParallelism = context.GetHostContext().GetService <IConfigurationStore>().GetSettings().MaxDedupParallelism; (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance .CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, maxParallelism, token); // Upload to blobstore var results = await BlobStoreUtils.UploadBatchToBlobstore(verbose, files, (level, uri, type) => new BuildArtifactActionRecord(level, uri, type, nameof(BlobUploadAsync), context), (str) => context.Output(str), dedupClient, clientTelemetry, token, enableReporting : true); // Associate with TFS context.Output(StringUtil.Loc("AssociateFiles")); var queue = new ConcurrentQueue <BlobFileInfo>(); foreach (var file in results.fileDedupIds) { queue.Enqueue(file); } // Start associate monitor var uploadFinished = new TaskCompletionSource <int>(); var associateMonitor = AssociateReportingAsync(context, files.Count(), uploadFinished, token); // Start parallel associate tasks. var parallelAssociateTasks = new List <Task <UploadResult> >(); for (int uploader = 0; uploader < concurrentUploads; uploader++) { parallelAssociateTasks.Add(AssociateAsync(context, queue, token)); } // Wait for parallel associate tasks to finish. await Task.WhenAll(parallelAssociateTasks); foreach (var associateTask in parallelAssociateTasks) { // record all failed files. uploadResult.AddUploadResult(await associateTask); } // Stop monitor task uploadFinished.SetResult(0); await associateMonitor; // report telemetry if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.PlanId), out var planId)) { planId = Guid.Empty; } if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.JobId), out var jobId)) { jobId = Guid.Empty; } await clientTelemetry.CommitTelemetryUpload(planId, jobId); } catch (SocketException e) { ExceptionsUtil.HandleSocketException(e, this._connection.Uri.ToString(), context.Warn); throw; } catch { var blobStoreHost = dedupClient.Client.BaseAddress.Host; var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform(); var warningMessage = StringUtil.Loc("BlobStoreUploadWarning", blobStoreHost, allowListLink); context.Warn(warningMessage); throw; } return(uploadResult); }