/// <summary> /// Determine whether a local package is the same as a package on an AzDO feed. /// </summary> /// <param name="localPackageFullPath"></param> /// <param name="packageContentUrl"></param> /// <param name="client"></param> /// <returns></returns> /// <remarks> /// Open a stream to the local file and an http request to the package. There are a couple possibilities: /// - The returned headers includes a content MD5 header, in which case we can /// hash the local file and just compare those. /// - No content MD5 hash, and the streams must be compared in blocks. This is a bit trickier to do efficiently, /// since we do not necessarily want to read all bytes if we can help it. Thus, we should compare in blocks. However, /// the streams make no gaurantee that they will return a full block each time when read operations are performed, so we /// must be sure to only compare the minimum number of bytes returned. /// </remarks> public static async Task <PackageFeedStatus> CompareLocalPackageToFeedPackage(string localPackageFullPath, string packageContentUrl, HttpClient client, TaskLoggingHelper log) { log.LogMessage($"Getting package content from {packageContentUrl} and comparing to {localPackageFullPath}"); PackageFeedStatus result = PackageFeedStatus.Unknown; ExponentialRetry RetryHandler = new ExponentialRetry { MaxAttempts = MaxRetries }; bool success = await RetryHandler.RunAsync(async attempt => { try { using (Stream localFileStream = File.OpenRead(localPackageFullPath)) using (HttpResponseMessage response = await client.GetAsync(packageContentUrl)) { response.EnsureSuccessStatusCode(); // Check the headers for content length and md5 bool md5HeaderAvailable = response.Headers.TryGetValues("Content-MD5", out var md5); bool lengthHeaderAvailable = response.Headers.TryGetValues("Content-Length", out var contentLength); if (lengthHeaderAvailable && long.Parse(contentLength.Single()) != localFileStream.Length) { log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different length than remote package '{packageContentUrl}'."); result = PackageFeedStatus.ExistsAndDifferent; return(true); } if (md5HeaderAvailable) { var localMD5 = AzureStorageUtils.CalculateMD5(localPackageFullPath); if (!localMD5.Equals(md5.Single(), StringComparison.OrdinalIgnoreCase)) { log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different MD5 hash than remote package '{packageContentUrl}'."); } result = PackageFeedStatus.ExistsAndDifferent; return(true); } const int BufferSize = 64 * 1024; // Otherwise, compare the streams var remoteStream = await response.Content.ReadAsStreamAsync(); var streamsMatch = await GeneralUtils.CompareStreamsAsync(localFileStream, remoteStream, BufferSize); result = streamsMatch ? PackageFeedStatus.ExistsAndIdenticalToLocal : PackageFeedStatus.ExistsAndDifferent; return(true); } } // String based comparison because the status code isn't exposed in HttpRequestException // see here: https://github.com/dotnet/runtime/issues/23648 catch (HttpRequestException e) { if (e.Message.Contains("404 (Not Found)")) { result = PackageFeedStatus.DoesNotExist; return(true); } // Retry this. Could be an http client timeout, 500, etc. return(false); } }); return(result); }
/// <summary> /// Parse out the input TargetFeedConfig into a dictionary of FeedConfig types /// </summary> public async Task ParseTargetFeedConfigAsync() { using (HttpClient httpClient = new HttpClient(new HttpClientHandler { CheckCertificateRevocationList = true })) { foreach (var fc in TargetFeedConfig) { string targetFeedUrl = fc.GetMetadata(nameof(Model.TargetFeedConfig.TargetURL)); string feedKey = fc.GetMetadata(nameof(Model.TargetFeedConfig.Token)); string type = fc.GetMetadata(nameof(Model.TargetFeedConfig.Type)); AssetSelection assetSelection = AssetSelection.All; bool isInternalFeed; bool isIsolatedFeed = false; bool isOverridableFeed = false; if (string.IsNullOrEmpty(targetFeedUrl) || string.IsNullOrEmpty(feedKey) || string.IsNullOrEmpty(type)) { Log.LogError($"Invalid FeedConfig entry. {nameof(Model.TargetFeedConfig.TargetURL)}='{targetFeedUrl}' {nameof(Model.TargetFeedConfig.Type)}='{type}' {nameof(Model.TargetFeedConfig.Token)}='{feedKey}'"); continue; } if (!targetFeedUrl.EndsWith(PublishingConstants.ExpectedFeedUrlSuffix)) { Log.LogError($"Exepcted that feed '{targetFeedUrl}' would end in {PublishingConstants.ExpectedFeedUrlSuffix}"); continue; } if (!Enum.TryParse <FeedType>(type, true, out FeedType feedType)) { Log.LogError($"Invalid feed config type '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(FeedType)))}"); continue; } string assetSelectionStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.AssetSelection)); if (!string.IsNullOrEmpty(assetSelectionStr)) { if (!Enum.TryParse <AssetSelection>(assetSelectionStr, true, out assetSelection)) { Log.LogError($"Invalid feed config asset selection '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(AssetSelection)))}"); continue; } } // To determine whether a feed is internal, we allow the user to // specify the value explicitly. string isInternalFeedStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.Internal)); if (!string.IsNullOrEmpty(isInternalFeedStr)) { if (!bool.TryParse(isInternalFeedStr, out isInternalFeed)) { Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.Internal)}' setting. Must be 'true' or 'false'."); continue; } } else { bool?isPublicFeed = await GeneralUtils.IsFeedPublicAsync(targetFeedUrl, httpClient, Log); if (!isPublicFeed.HasValue) { continue; } else { isInternalFeed = !isPublicFeed.Value; } } string isIsolatedFeedStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.Isolated)); if (!string.IsNullOrEmpty(isIsolatedFeedStr)) { if (!bool.TryParse(isIsolatedFeedStr, out isIsolatedFeed)) { Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.Isolated)}' setting. Must be 'true' or 'false'."); continue; } } string allowOverwriteOnFeed = fc.GetMetadata(nameof(Model.TargetFeedConfig.AllowOverwrite)); if (!string.IsNullOrEmpty(allowOverwriteOnFeed)) { if (!bool.TryParse(allowOverwriteOnFeed, out isOverridableFeed)) { Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.AllowOverwrite)}' setting. Must be 'true' or 'false'."); continue; } } string latestLinkShortUrlPrefix = fc.GetMetadata(nameof(Model.TargetFeedConfig.LatestLinkShortUrlPrefixes)); if (!string.IsNullOrEmpty(latestLinkShortUrlPrefix)) { // Verify other inputs are provided if (string.IsNullOrEmpty(AkaMSClientId) || string.IsNullOrEmpty(AkaMSClientSecret) || string.IsNullOrEmpty(AkaMSTenant) || string.IsNullOrEmpty(AkaMsOwners) || string.IsNullOrEmpty(AkaMSCreatedBy)) { Log.LogError($"If a short url path is provided, please provide {nameof(AkaMSClientId)}, {nameof(AkaMSClientSecret)}, " + $"{nameof(AkaMSTenant)}, {nameof(AkaMsOwners)}, {nameof(AkaMSCreatedBy)}"); continue; } // Set up the link manager if it hasn't already been done if (LinkManager == null) { LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, AkaMSCreatedBy, AkaMsOwners, Log); } } if (!Enum.TryParse(fc.ItemSpec, ignoreCase: true, out TargetFeedContentType categoryKey)) { Log.LogError($"Invalid target feed config category '{fc.ItemSpec}'."); } if (!FeedConfigs.TryGetValue(categoryKey, out _)) { FeedConfigs[categoryKey] = new HashSet <TargetFeedConfig>(); } TargetFeedConfig feedConfig = new TargetFeedConfig( contentType: categoryKey, targetURL: targetFeedUrl, type: feedType, token: feedKey, latestLinkShortUrlPrefixes: new List <string>() { latestLinkShortUrlPrefix }, assetSelection: assetSelection, isolated: isIsolatedFeed, @internal: isInternalFeed, allowOverwrite: isOverridableFeed); CheckForInternalBuildsOnPublicFeeds(feedConfig); FeedConfigs[categoryKey].Add(feedConfig); } } }
public BuildModel CreateModelFromItems( ITaskItem[] artifacts, ITaskItem[] itemsToSign, ITaskItem[] strongNameSignInfo, ITaskItem[] fileSignInfo, ITaskItem[] fileExtensionSignInfo, ITaskItem[] certificatesSignInfo, string buildId, string[] manifestBuildData, string repoUri, string repoBranch, string repoCommit, bool isStableBuild, PublishingInfraVersion publishingVersion, bool isReleaseOnlyPackageVersion) { if (artifacts == null) { throw new ArgumentNullException(nameof(artifacts)); } var blobArtifacts = new List <BlobArtifactModel>(); var packageArtifacts = new List <PackageArtifactModel>(); foreach (var artifact in artifacts) { if (string.Equals(artifact.GetMetadata("ExcludeFromManifest"), "true", StringComparison.OrdinalIgnoreCase)) { continue; } var isSymbolsPackage = GeneralUtils.IsSymbolPackage(artifact.ItemSpec); if (artifact.ItemSpec.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase) && !isSymbolsPackage) { packageArtifacts.Add(_packageArtifactModelFactory.CreatePackageArtifactModel(artifact)); } else { if (isSymbolsPackage) { string fileName = Path.GetFileName(artifact.ItemSpec); artifact.SetMetadata("RelativeBlobPath", $"{AssetsVirtualDir}symbols/{fileName}"); } blobArtifacts.Add(_blobArtifactModelFactory.CreateBlobArtifactModel(artifact)); } } var signingInfoModel = _signingInformationModelFactory.CreateSigningInformationModelFromItems( itemsToSign, strongNameSignInfo, fileSignInfo, fileExtensionSignInfo, certificatesSignInfo, blobArtifacts, packageArtifacts); var buildModel = CreateModel( blobArtifacts, packageArtifacts, buildId, manifestBuildData, repoUri, repoBranch, repoCommit, isStableBuild, publishingVersion, isReleaseOnlyPackageVersion, signingInformationModel: signingInfoModel); return(buildModel); }