/// <summary>
/// Determine whether a local package is the same as a package on an AzDO feed.
/// </summary>
/// <param name="localPackageFullPath"></param>
/// <param name="packageContentUrl"></param>
/// <param name="client"></param>
/// <returns></returns>
/// <remarks>
/// Open a stream to the local file and an http request to the package. There are a couple possibilities:
/// - The returned headers includes a content MD5 header, in which case we can
/// hash the local file and just compare those.
/// - No content MD5 hash, and the streams must be compared in blocks. This is a bit trickier to do efficiently,
/// since we do not necessarily want to read all bytes if we can help it. Thus, we should compare in blocks. However,
/// the streams make no gaurantee that they will return a full block each time when read operations are performed, so we
/// must be sure to only compare the minimum number of bytes returned.
/// </remarks>
public static async Task <PackageFeedStatus> CompareLocalPackageToFeedPackage(string localPackageFullPath, string packageContentUrl, HttpClient client, TaskLoggingHelper log)
{
log.LogMessage($"Getting package content from {packageContentUrl} and comparing to {localPackageFullPath}");
PackageFeedStatus result = PackageFeedStatus.Unknown;
ExponentialRetry RetryHandler = new ExponentialRetry
{
MaxAttempts = MaxRetries
};
bool success = await RetryHandler.RunAsync(async attempt =>
{
try
{
using (Stream localFileStream = File.OpenRead(localPackageFullPath))
using (HttpResponseMessage response = await client.GetAsync(packageContentUrl))
{
response.EnsureSuccessStatusCode();
// Check the headers for content length and md5
bool md5HeaderAvailable = response.Headers.TryGetValues("Content-MD5", out var md5);
bool lengthHeaderAvailable = response.Headers.TryGetValues("Content-Length", out var contentLength);
if (lengthHeaderAvailable && long.Parse(contentLength.Single()) != localFileStream.Length)
{
log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different length than remote package '{packageContentUrl}'.");
result = PackageFeedStatus.ExistsAndDifferent;
return(true);
}
if (md5HeaderAvailable)
{
var localMD5 = AzureStorageUtils.CalculateMD5(localPackageFullPath);
if (!localMD5.Equals(md5.Single(), StringComparison.OrdinalIgnoreCase))
{
log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different MD5 hash than remote package '{packageContentUrl}'.");
}
result = PackageFeedStatus.ExistsAndDifferent;
return(true);
}
const int BufferSize = 64 * 1024;
// Otherwise, compare the streams
var remoteStream = await response.Content.ReadAsStreamAsync();
var streamsMatch = await GeneralUtils.CompareStreamsAsync(localFileStream, remoteStream, BufferSize);
result = streamsMatch ? PackageFeedStatus.ExistsAndIdenticalToLocal : PackageFeedStatus.ExistsAndDifferent;
return(true);
}
}
// String based comparison because the status code isn't exposed in HttpRequestException
// see here: https://github.com/dotnet/runtime/issues/23648
catch (HttpRequestException e)
{
if (e.Message.Contains("404 (Not Found)"))
{
result = PackageFeedStatus.DoesNotExist;
return(true);
}
// Retry this. Could be an http client timeout, 500, etc.
return(false);
}
});
return(result);
}
/// <summary>
/// Parse out the input TargetFeedConfig into a dictionary of FeedConfig types
/// </summary>
public async Task ParseTargetFeedConfigAsync()
{
using (HttpClient httpClient = new HttpClient(new HttpClientHandler {
CheckCertificateRevocationList = true
}))
{
foreach (var fc in TargetFeedConfig)
{
string targetFeedUrl = fc.GetMetadata(nameof(Model.TargetFeedConfig.TargetURL));
string feedKey = fc.GetMetadata(nameof(Model.TargetFeedConfig.Token));
string type = fc.GetMetadata(nameof(Model.TargetFeedConfig.Type));
AssetSelection assetSelection = AssetSelection.All;
bool isInternalFeed;
bool isIsolatedFeed = false;
bool isOverridableFeed = false;
if (string.IsNullOrEmpty(targetFeedUrl) ||
string.IsNullOrEmpty(feedKey) ||
string.IsNullOrEmpty(type))
{
Log.LogError($"Invalid FeedConfig entry. {nameof(Model.TargetFeedConfig.TargetURL)}='{targetFeedUrl}' {nameof(Model.TargetFeedConfig.Type)}='{type}' {nameof(Model.TargetFeedConfig.Token)}='{feedKey}'");
continue;
}
if (!targetFeedUrl.EndsWith(PublishingConstants.ExpectedFeedUrlSuffix))
{
Log.LogError($"Exepcted that feed '{targetFeedUrl}' would end in {PublishingConstants.ExpectedFeedUrlSuffix}");
continue;
}
if (!Enum.TryParse <FeedType>(type, true, out FeedType feedType))
{
Log.LogError($"Invalid feed config type '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(FeedType)))}");
continue;
}
string assetSelectionStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.AssetSelection));
if (!string.IsNullOrEmpty(assetSelectionStr))
{
if (!Enum.TryParse <AssetSelection>(assetSelectionStr, true, out assetSelection))
{
Log.LogError($"Invalid feed config asset selection '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(AssetSelection)))}");
continue;
}
}
// To determine whether a feed is internal, we allow the user to
// specify the value explicitly.
string isInternalFeedStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.Internal));
if (!string.IsNullOrEmpty(isInternalFeedStr))
{
if (!bool.TryParse(isInternalFeedStr, out isInternalFeed))
{
Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.Internal)}' setting. Must be 'true' or 'false'.");
continue;
}
}
else
{
bool?isPublicFeed = await GeneralUtils.IsFeedPublicAsync(targetFeedUrl, httpClient, Log);
if (!isPublicFeed.HasValue)
{
continue;
}
else
{
isInternalFeed = !isPublicFeed.Value;
}
}
string isIsolatedFeedStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.Isolated));
if (!string.IsNullOrEmpty(isIsolatedFeedStr))
{
if (!bool.TryParse(isIsolatedFeedStr, out isIsolatedFeed))
{
Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.Isolated)}' setting. Must be 'true' or 'false'.");
continue;
}
}
string allowOverwriteOnFeed = fc.GetMetadata(nameof(Model.TargetFeedConfig.AllowOverwrite));
if (!string.IsNullOrEmpty(allowOverwriteOnFeed))
{
if (!bool.TryParse(allowOverwriteOnFeed, out isOverridableFeed))
{
Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.AllowOverwrite)}' setting. Must be 'true' or 'false'.");
continue;
}
}
string latestLinkShortUrlPrefix = fc.GetMetadata(nameof(Model.TargetFeedConfig.LatestLinkShortUrlPrefixes));
if (!string.IsNullOrEmpty(latestLinkShortUrlPrefix))
{
// Verify other inputs are provided
if (string.IsNullOrEmpty(AkaMSClientId) ||
string.IsNullOrEmpty(AkaMSClientSecret) ||
string.IsNullOrEmpty(AkaMSTenant) ||
string.IsNullOrEmpty(AkaMsOwners) ||
string.IsNullOrEmpty(AkaMSCreatedBy))
{
Log.LogError($"If a short url path is provided, please provide {nameof(AkaMSClientId)}, {nameof(AkaMSClientSecret)}, " +
$"{nameof(AkaMSTenant)}, {nameof(AkaMsOwners)}, {nameof(AkaMSCreatedBy)}");
continue;
}
// Set up the link manager if it hasn't already been done
if (LinkManager == null)
{
LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, AkaMSCreatedBy, AkaMsOwners, Log);
}
}
if (!Enum.TryParse(fc.ItemSpec, ignoreCase: true, out TargetFeedContentType categoryKey))
{
Log.LogError($"Invalid target feed config category '{fc.ItemSpec}'.");
}
if (!FeedConfigs.TryGetValue(categoryKey, out _))
{
FeedConfigs[categoryKey] = new HashSet <TargetFeedConfig>();
}
TargetFeedConfig feedConfig = new TargetFeedConfig(
contentType: categoryKey,
targetURL: targetFeedUrl,
type: feedType,
token: feedKey,
latestLinkShortUrlPrefixes: new List <string>()
{
latestLinkShortUrlPrefix
},
assetSelection: assetSelection,
isolated: isIsolatedFeed,
@internal: isInternalFeed,
allowOverwrite: isOverridableFeed);
CheckForInternalBuildsOnPublicFeeds(feedConfig);
FeedConfigs[categoryKey].Add(feedConfig);
}
}
}
public BuildModel CreateModelFromItems(
ITaskItem[] artifacts,
ITaskItem[] itemsToSign,
ITaskItem[] strongNameSignInfo,
ITaskItem[] fileSignInfo,
ITaskItem[] fileExtensionSignInfo,
ITaskItem[] certificatesSignInfo,
string buildId,
string[] manifestBuildData,
string repoUri,
string repoBranch,
string repoCommit,
bool isStableBuild,
PublishingInfraVersion publishingVersion,
bool isReleaseOnlyPackageVersion)
{
if (artifacts == null)
{
throw new ArgumentNullException(nameof(artifacts));
}
var blobArtifacts = new List <BlobArtifactModel>();
var packageArtifacts = new List <PackageArtifactModel>();
foreach (var artifact in artifacts)
{
if (string.Equals(artifact.GetMetadata("ExcludeFromManifest"), "true", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var isSymbolsPackage = GeneralUtils.IsSymbolPackage(artifact.ItemSpec);
if (artifact.ItemSpec.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase) && !isSymbolsPackage)
{
packageArtifacts.Add(_packageArtifactModelFactory.CreatePackageArtifactModel(artifact));
}
else
{
if (isSymbolsPackage)
{
string fileName = Path.GetFileName(artifact.ItemSpec);
artifact.SetMetadata("RelativeBlobPath", $"{AssetsVirtualDir}symbols/{fileName}");
}
blobArtifacts.Add(_blobArtifactModelFactory.CreateBlobArtifactModel(artifact));
}
}
var signingInfoModel = _signingInformationModelFactory.CreateSigningInformationModelFromItems(
itemsToSign, strongNameSignInfo, fileSignInfo, fileExtensionSignInfo,
certificatesSignInfo, blobArtifacts, packageArtifacts);
var buildModel = CreateModel(
blobArtifacts,
packageArtifacts,
buildId,
manifestBuildData,
repoUri,
repoBranch,
repoCommit,
isStableBuild,
publishingVersion,
isReleaseOnlyPackageVersion,
signingInformationModel: signingInfoModel);
return(buildModel);
}
