private async Task PushPackagesToFeed(string assetsFolder, string feedUrl)
{
string packagesFolder = Path.Combine(assetsFolder, "packages");
TargetFeedConfig targetFeedConfig = new TargetFeedConfig(TargetFeedContentType.Package, feedUrl, FeedType.AzDoNugetFeed, AzureDevOpsPersonalAccessToken);
HashSet <PackageIdentity> packagesToPublish = new HashSet <PackageIdentity>(
Directory.GetFiles(packagesFolder).Select(packagePath =>
{
using (BinaryReader reader = new BinaryReader(File.Open(packagePath, FileMode.Open)))
{
PackageArchiveReader packageReader = new PackageArchiveReader(reader.BaseStream);
return(packageReader.NuspecReader.GetIdentity());
}
}));
await PushNugetPackagesAsync <PackageIdentity>(packagesToPublish, targetFeedConfig, 5,
async (feed, httpClient, package, feedAccount, feedVisibility, feedName) =>
{
string localPackagePath = Path.Combine(packagesFolder, $"{package.Id}.{package.Version}.nupkg");
if (!File.Exists(localPackagePath))
{
Log.LogError($"Could not locate '{package.Id}.{package.Version}' at '{localPackagePath}'");
return;
}
await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version.ToString(), feedAccount, feedVisibility, feedName);
});
}
/// <summary>
/// Get the short url for a blob.
/// </summary>
/// <param name="feedConfig">Feed configuration</param>
/// <param name="blob">Blob</param>
/// <returns>Short url prefix for the blob.</returns>
/// <remarks>
public string GetLatestShortUrlForBlob(TargetFeedConfig feedConfig, BlobArtifactModel blob, bool flatten)
{
string blobIdWithoutVersions = VersionIdentifier.RemoveVersions(blob.Id);
if (flatten)
{
blobIdWithoutVersions = Path.GetFileName(blobIdWithoutVersions);
}
return(Path.Combine(feedConfig.LatestLinkShortUrlPrefix, blobIdWithoutVersions).Replace("\\", "/"));
}
public virtual IAssetPublisher CreateAssetPublisher(TargetFeedConfig feedConfig, PublishArtifactsInManifestBase task)
{
switch (feedConfig.Type)
{
case FeedType.AzDoNugetFeed:
return(new AzureDevOpsNugetFeedAssetPublisher(_log, feedConfig.TargetURL, feedConfig.Token, task));
case FeedType.AzureStorageFeed:
var action = new BlobFeedAction(feedConfig.TargetURL, feedConfig.Token, _log);
return(new AzureStorageFeedAssetPublisher(action.AccountName, action.AccountKey, action.ContainerName, _log));
case FeedType.AzureStorageContainer:
return(new AzureStorageContainerAssetPublisher(new Uri(feedConfig.TargetURL), _log));
default:
throw new NotImplementedException();
}
}
public async System.Threading.Tasks.Task CreateOrUpdateLatestLinksAsync(
HashSet <BlobArtifactModel> blobsToPublish,
TargetFeedConfig feedConfig,
int expectedSuffixLength)
{
if (string.IsNullOrEmpty(feedConfig.LatestLinkShortUrlPrefix))
{
return;
}
Logger.LogMessage(MessageImportance.High, "\nThe following aka.ms links for blobs will be created:");
IEnumerable <AkaMSLink> linksToCreate = blobsToPublish
.Where(blob => !feedConfig.FilenamesToExclude.Contains(Path.GetFileName(blob.Id)))
.Select(blob =>
{
// Strip away the feed expected suffix (index.json) and append on the
// blob path.
string actualTargetUrl = feedConfig.TargetURL.Substring(0,
feedConfig.TargetURL.Length - expectedSuffixLength) + blob.Id;
// The dotnetcli storage account is in a single datacenter in the US and thus download
// times can be painful elsewhere. The CDN helps with this thefore we point the target
// of the aka.ms links to the CDN.
actualTargetUrl = actualTargetUrl.Replace("//dotnetcli.blob.core.windows.net/", "//dotnetcli.azureedge.net/");
AkaMSLink newLink = new AkaMSLink
{
ShortUrl = GetLatestShortUrlForBlob(feedConfig, blob, feedConfig.Flatten),
TargetUrl = actualTargetUrl
};
Logger.LogMessage(MessageImportance.High, $" {Path.GetFileName(blob.Id)}");
Logger.LogMessage(MessageImportance.High, $" aka.ms/{newLink.ShortUrl} -> {newLink.TargetUrl}");
return(newLink);
}).ToList();
await LinkManager.CreateOrUpdateLinksAsync(linksToCreate, AkaMsOwners, AkaMSCreatedBy, AkaMSGroupOwner, true);
}
public async Task PushNugetPackageTestsAsync(int pushAttemptsBeforeSuccess, bool packageAlreadyOnFeed, bool localPackageMatchesFeed, bool expectedFailure = false)
{
// Setup
var buildEngine = new MockBuildEngine();
// May as well check that the exe is plumbed through from the task.
string fakeNugetExeName = $"{Path.GetRandomFileName()}.exe";
int timesNugetExeCalled = 0;
// Functionality is the same as this is in the base class, create a v2 object to test.
var task = new PublishArtifactsInManifestV2
{
InternalBuild = true,
BuildEngine = buildEngine,
NugetPath = fakeNugetExeName,
MaxRetryCount = 5, // In case the default changes, lock to 5 so the test data works
RetryDelayMilliseconds = 10 // retry faster in test
};
TargetFeedConfig config = new TargetFeedConfig(TargetFeedContentType.Package, "testUrl", FeedType.AzDoNugetFeed, "tokenValue");
Func <string, string, HttpClient, MsBuildUtils.TaskLoggingHelper, Task <PackageFeedStatus> > testCompareLocalPackage = async(string localPackageFullPath, string packageContentUrl, HttpClient client, MsBuildUtils.TaskLoggingHelper log) =>
{
await(Task.Delay(10)); // To make this actually async
Debug.WriteLine($"Called mocked CompareLocalPackageToFeedPackage() : localPackageFullPath = {localPackageFullPath}, packageContentUrl = {packageContentUrl}");
if (packageAlreadyOnFeed)
{
return(localPackageMatchesFeed ? PackageFeedStatus.ExistsAndIdenticalToLocal : PackageFeedStatus.ExistsAndDifferent);
}
else
{
return(PackageFeedStatus.DoesNotExist);
}
};
Func <string, string, Task <ProcessExecutionResult> > testRunAndLogProcess = (string fakeExePath, string fakeExeArgs) =>
{
Debug.WriteLine($"Called mocked RunProcessAndGetOutputs() : ExePath = {fakeExePath}, ExeArgs = {fakeExeArgs}");
fakeNugetExeName.Should().Be(fakeExePath);
ProcessExecutionResult result = new ProcessExecutionResult()
{
StandardError = "fake stderr", StandardOut = "fake stdout"
};
timesNugetExeCalled++;
if (timesNugetExeCalled >= pushAttemptsBeforeSuccess)
{
result.ExitCode = 0;
}
else
{
result.ExitCode = 1;
}
return(Task.FromResult(result));
};
await task.PushNugetPackageAsync(
config,
null,
"localPackageLocation",
"1234",
"version",
"feedaccount",
"feedvisibility",
"feedname",
testCompareLocalPackage,
testRunAndLogProcess);
if (!expectedFailure && localPackageMatchesFeed)
{
// Successful retry scenario; make sure we ran the # of retries we thought.
timesNugetExeCalled.Should().BeLessOrEqualTo(task.MaxRetryCount);
}
expectedFailure.Should().Be(task.Log.HasLoggedErrors);
}
/// <summary>
/// Parse out the input TargetFeedConfig into a dictionary of FeedConfig types
/// </summary>
public async Task ParseTargetFeedConfigAsync()
{
using (HttpClient httpClient = new HttpClient(new HttpClientHandler {
CheckCertificateRevocationList = true
}))
{
foreach (var fc in TargetFeedConfig)
{
string targetFeedUrl = fc.GetMetadata(nameof(Model.TargetFeedConfig.TargetURL));
string feedKey = fc.GetMetadata(nameof(Model.TargetFeedConfig.Token));
string type = fc.GetMetadata(nameof(Model.TargetFeedConfig.Type));
AssetSelection assetSelection = AssetSelection.All;
bool isInternalFeed;
bool isIsolatedFeed = false;
bool isOverridableFeed = false;
if (string.IsNullOrEmpty(targetFeedUrl) ||
string.IsNullOrEmpty(feedKey) ||
string.IsNullOrEmpty(type))
{
Log.LogError($"Invalid FeedConfig entry. {nameof(Model.TargetFeedConfig.TargetURL)}='{targetFeedUrl}' {nameof(Model.TargetFeedConfig.Type)}='{type}' {nameof(Model.TargetFeedConfig.Token)}='{feedKey}'");
continue;
}
if (!targetFeedUrl.EndsWith(PublishingConstants.ExpectedFeedUrlSuffix))
{
Log.LogError($"Exepcted that feed '{targetFeedUrl}' would end in {PublishingConstants.ExpectedFeedUrlSuffix}");
continue;
}
if (!Enum.TryParse <FeedType>(type, true, out FeedType feedType))
{
Log.LogError($"Invalid feed config type '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(FeedType)))}");
continue;
}
string assetSelectionStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.AssetSelection));
if (!string.IsNullOrEmpty(assetSelectionStr))
{
if (!Enum.TryParse <AssetSelection>(assetSelectionStr, true, out assetSelection))
{
Log.LogError($"Invalid feed config asset selection '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(AssetSelection)))}");
continue;
}
}
// To determine whether a feed is internal, we allow the user to
// specify the value explicitly.
string isInternalFeedStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.Internal));
if (!string.IsNullOrEmpty(isInternalFeedStr))
{
if (!bool.TryParse(isInternalFeedStr, out isInternalFeed))
{
Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.Internal)}' setting. Must be 'true' or 'false'.");
continue;
}
}
else
{
bool?isPublicFeed = await GeneralUtils.IsFeedPublicAsync(targetFeedUrl, httpClient, Log);
if (!isPublicFeed.HasValue)
{
continue;
}
else
{
isInternalFeed = !isPublicFeed.Value;
}
}
string isIsolatedFeedStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.Isolated));
if (!string.IsNullOrEmpty(isIsolatedFeedStr))
{
if (!bool.TryParse(isIsolatedFeedStr, out isIsolatedFeed))
{
Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.Isolated)}' setting. Must be 'true' or 'false'.");
continue;
}
}
string allowOverwriteOnFeed = fc.GetMetadata(nameof(Model.TargetFeedConfig.AllowOverwrite));
if (!string.IsNullOrEmpty(allowOverwriteOnFeed))
{
if (!bool.TryParse(allowOverwriteOnFeed, out isOverridableFeed))
{
Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.AllowOverwrite)}' setting. Must be 'true' or 'false'.");
continue;
}
}
string latestLinkShortUrlPrefix = fc.GetMetadata(nameof(Model.TargetFeedConfig.LatestLinkShortUrlPrefixes));
if (!string.IsNullOrEmpty(latestLinkShortUrlPrefix))
{
// Verify other inputs are provided
if (string.IsNullOrEmpty(AkaMSClientId) ||
string.IsNullOrEmpty(AkaMSClientSecret) ||
string.IsNullOrEmpty(AkaMSTenant) ||
string.IsNullOrEmpty(AkaMsOwners) ||
string.IsNullOrEmpty(AkaMSCreatedBy))
{
Log.LogError($"If a short url path is provided, please provide {nameof(AkaMSClientId)}, {nameof(AkaMSClientSecret)}, " +
$"{nameof(AkaMSTenant)}, {nameof(AkaMsOwners)}, {nameof(AkaMSCreatedBy)}");
continue;
}
// Set up the link manager if it hasn't already been done
if (LinkManager == null)
{
LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, AkaMSCreatedBy, AkaMsOwners, Log);
}
}
if (!Enum.TryParse(fc.ItemSpec, ignoreCase: true, out TargetFeedContentType categoryKey))
{
Log.LogError($"Invalid target feed config category '{fc.ItemSpec}'.");
}
if (!FeedConfigs.TryGetValue(categoryKey, out _))
{
FeedConfigs[categoryKey] = new HashSet <TargetFeedConfig>();
}
TargetFeedConfig feedConfig = new TargetFeedConfig(
contentType: categoryKey,
targetURL: targetFeedUrl,
type: feedType,
token: feedKey,
latestLinkShortUrlPrefixes: new List <string>()
{
latestLinkShortUrlPrefix
},
assetSelection: assetSelection,
isolated: isIsolatedFeed,
@internal: isInternalFeed,
allowOverwrite: isOverridableFeed);
CheckForInternalBuildsOnPublicFeeds(feedConfig);
FeedConfigs[categoryKey].Add(feedConfig);
}
}
}
