public override async Task <int> ExecuteAsync()
{
Local local = new Local(Logger);
try
{
IEnumerable <DependencyDetail> dependencies = await local.GetDependenciesAsync(_options.Name);
if (!string.IsNullOrEmpty(_options.Name))
{
DependencyDetail dependency = dependencies.Where(d => d.Name.Equals(_options.Name, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
if (dependency == null)
{
throw new Exception($"A dependency with name '{_options.Name}' was not found...");
}
LogDependency(dependency);
}
foreach (DependencyDetail dependency in dependencies)
{
LogDependency(dependency);
Console.WriteLine();
}
return(Constants.SuccessCode);
}
catch (Exception exc)
{
if (!string.IsNullOrEmpty(_options.Name))
{
Logger.LogError(exc, $"Something failed while querying for local dependency '{_options.Name}'.");
}
else
{
Logger.LogError(exc, "Something failed while querying for local dependencies.");
}
return(Constants.ErrorCode);
}
}
public void ValidateGraphWithNoChildNodes()
{
DependencyDetail dependencyDetail = new DependencyDetail
{
Name = "Base.Asset",
RepoUri = "RepoX",
Commit = "shaX",
Version = "1.0.0"
};
DependencyTestDriver.GetGraphAndCompare("DependencyGraph", async driver =>
{
return(await driver.GetDependencyGraph(dependencyDetail));
},
GetExpectedDependencyGraphAsync,
dependencyDetail,
DefaultOutputFileName,
true);
}
public void ValidateIncompleteFullGraph()
{
DependencyDetail dependencyDetail = new DependencyDetail
{
Name = "Base.Asset",
RepoUri = "RepoA",
Commit = "shaA",
Version = "1.0.0"
};
DependencyTestDriver.GetGraphAndCompare("DependencyGraph", async driver =>
{
return(await driver.GetDependencyGraph(dependencyDetail));
},
GetExpectedDependencyGraphAsync,
dependencyDetail,
"output1.xml",
false);
}
private async Task <int> NonCoherencyUpdatesForBuildAsync(
Build build,
IRemote barOnlyRemote,
List <DependencyDetail> currentDependencies,
List <DependencyDetail> dependenciesToUpdate)
{
IEnumerable <AssetData> assetData = build.Assets.Select(
a => new AssetData(a.NonShipping)
{
Name = a.Name,
Version = a.Version
});
string repository = build.GitHubRepository ?? build.AzureDevOpsRepository;
// Now determine what needs to be updated.
List <DependencyUpdate> updates = await barOnlyRemote.
GetRequiredNonCoherencyUpdatesAsync(repository, build.Commit, assetData, currentDependencies);
foreach (DependencyUpdate update in updates)
{
DependencyDetail from = update.From;
DependencyDetail to = update.To;
// Print out what we are going to do.
Console.WriteLine($"Updating '{from.Name}': '{from.Version}' => '{to.Version}'"
+ $" (from build '{build.AzureDevOpsBuildNumber}' of '{repository}')");
// Replace in the current dependencies list so the correct data can be used in coherency updates.
currentDependencies.Remove(from);
currentDependencies.Add(to);
// Final list of dependencies to update
dependenciesToUpdate.Add(to);
}
return(Constants.SuccessCode);
}
public async Task CoherencyUpdateTests8()
{
// Initialize
var barClientMock = new Mock <IBarClient>();
Remote remote = new Remote(null, barClientMock.Object, NullLogger.Instance);
// Mock the remote used by build dependency graph to gather dependency details.
var dependencyGraphRemoteMock = new Mock <IRemote>();
// Always return the main remote.
var remoteFactoryMock = new Mock <IRemoteFactory>();
remoteFactoryMock.Setup(m => m.GetRemoteAsync(It.IsAny <string>(), It.IsAny <ILogger>())).ReturnsAsync(dependencyGraphRemoteMock.Object);
remoteFactoryMock.Setup(m => m.GetBarOnlyRemoteAsync(It.IsAny <ILogger>())).ReturnsAsync(remote);
List <DependencyDetail> existingDetails = new List <DependencyDetail>();
DependencyDetail depA = AddDependency(existingDetails, "depA", "v1", "repoA", "commit1", pinned: false);
DependencyDetail depB = AddDependency(existingDetails, "depB", "v3", "repoB", "commit1", pinned: false, coherentParent: "depA");
List <AssetData> assets = new List <AssetData>()
{
new AssetData(false)
{
Name = "depA", Version = "v2"
},
new AssetData(false)
{
Name = "depB", Version = "v5"
}
};
BuildProducesAssets(barClientMock, "repoA", "commit2", new List <(string name, string version)>
{
("depA", "v2")
});
List <DependencyDetail> repoADeps = new List <DependencyDetail>();
AddDependency(repoADeps, "depC", "v10", "repoB", "commit5", pinned: false);
RepoHasDependencies(dependencyGraphRemoteMock, "repoA", "commit2", repoADeps);
BuildProducesAssets(barClientMock, "repoB", "commit5", new List <(string name, string version)>
{
("depC", "v10")
});
List <DependencyUpdate> nonCoherencyUpdates =
await remote.GetRequiredNonCoherencyUpdatesAsync("repoA", "commit2", assets, existingDetails);
Assert.Collection(nonCoherencyUpdates,
u =>
{
Assert.Equal(depA, u.From);
Assert.Equal("v2", u.To.Version);
});
// Update the current dependency details with the non coherency updates
UpdateCurrentDependencies(existingDetails, nonCoherencyUpdates);
await Assert.ThrowsAsync <DarcException>(() => remote.GetRequiredCoherencyUpdatesAsync(
existingDetails, remoteFactoryMock.Object));
}
internal bool HasDependencyOn(DependencyDetail dep)
{
return(HasDependencyOn(dep.RepoUri));
}
internal void AddDependency(DependencyDetail dep)
{
this.AddDependency(GetDependency(dep));
}
private StrippedDependency(DependencyDetail d) : this(d.RepoUri, d.Commit)
{
}
internal static StrippedDependency GetDependency(DependencyDetail d)
{
return(GetDependency(d.RepoUri, d.Commit));
}
private async Task <int> CoherencyUpdatesAsync(
IRemote barOnlyRemote,
IRemoteFactory remoteFactory,
List <DependencyDetail> currentDependencies,
List <DependencyDetail> dependenciesToUpdate)
{
Console.WriteLine("Checking for coherency updates...");
CoherencyMode coherencyMode = CoherencyMode.Strict;
if (_options.LegacyCoherency)
{
coherencyMode = CoherencyMode.Legacy;
}
else
{
Console.WriteLine("Using 'Strict' coherency mode. If this fails, a second attempt utilizing 'Legacy' Coherency mode will be made.");
}
List <DependencyUpdate> coherencyUpdates = null;
bool updateSucceeded = false;
try
{
// Now run a coherency update based on the current set of dependencies updated from the previous pass.
coherencyUpdates = await barOnlyRemote.GetRequiredCoherencyUpdatesAsync(currentDependencies, remoteFactory, coherencyMode);
updateSucceeded = true;
}
catch (DarcCoherencyException e)
{
PrettyPrintCoherencyErrors(e);
}
if (coherencyMode == CoherencyMode.Strict && !updateSucceeded)
{
Console.WriteLine("Attempting fallback to Legacy coherency");
try
{
// Now run a coherency update based on the current set of dependencies updated from the previous pass.
coherencyUpdates = await barOnlyRemote.GetRequiredCoherencyUpdatesAsync(currentDependencies, remoteFactory, CoherencyMode.Legacy);
Console.WriteLine("... Legacy-mode fallback worked");
updateSucceeded = true;
}
catch (DarcCoherencyException e)
{
PrettyPrintCoherencyErrors(e);
}
}
if (!updateSucceeded)
{
return(Constants.ErrorCode);
}
foreach (DependencyUpdate dependencyUpdate in coherencyUpdates)
{
DependencyDetail from = dependencyUpdate.From;
DependencyDetail to = dependencyUpdate.To;
DependencyDetail coherencyParent = currentDependencies.First(d =>
d.Name.Equals(from.CoherentParentDependencyName, StringComparison.OrdinalIgnoreCase));
// Print out what we are going to do.
Console.WriteLine($"Updating '{from.Name}': '{from.Version}' => '{to.Version}' " +
$"to ensure coherency with {from.CoherentParentDependencyName}@{coherencyParent.Version}");
// Final list of dependencies to update
dependenciesToUpdate.Add(to);
}
return(Constants.SuccessCode);
}
public ScenarioTests_GitHubFlow()
{
using TestParameters parameters = TestParameters.GetAsync().Result;
SetTestParameters(parameters);
source1Assets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0");
source2Assets = GetAssetData("Pizza", "3.1.0", "Hamburger", "4.1.0");
source1AssetsUpdated = GetAssetData("Foo", "1.17.0", "Bar", "2.17.0");
childSourceBuildAssets = GetAssetData("Baz", "1.3.0", "Bop", "1.0");
childSourceAssets = GetSingleAssetData("Baz", "1.3.0");
expectedDependenciesSource1 = new List <DependencyDetail>();
string sourceRepoUri = GetRepoUrl(TestRepository.TestRepo1Name);
DependencyDetail foo = new DependencyDetail
{
Name = "Foo",
Version = "1.1.0",
RepoUri = sourceRepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedDependenciesSource1.Add(foo);
DependencyDetail bar = new DependencyDetail
{
Name = "Bar",
Version = "2.1.0",
RepoUri = sourceRepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedDependenciesSource1.Add(bar);
expectedDependenciesSource2 = new List <DependencyDetail>();
string source2RepoUri = GetRepoUrl(TestRepository.TestRepo3Name);
DependencyDetail pizza = new DependencyDetail
{
Name = "Pizza",
Version = "3.1.0",
RepoUri = source2RepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedDependenciesSource2.Add(pizza);
DependencyDetail hamburger = new DependencyDetail
{
Name = "Hamburger",
Version = "4.1.0",
RepoUri = source2RepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedDependenciesSource2.Add(hamburger);
expectedDependenciesSource1Updated = new List <DependencyDetail>();
DependencyDetail fooUpdated = new DependencyDetail
{
Name = "Foo",
Version = "1.1.0",
RepoUri = sourceRepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedDependenciesSource1Updated.Add(fooUpdated);
DependencyDetail barUpdated = new DependencyDetail
{
Name = "Bar",
Version = "2.1.0",
RepoUri = sourceRepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedDependenciesSource1Updated.Add(barUpdated);
expectedCoherencyDependencies = new List <DependencyDetail>();
DependencyDetail baz = new DependencyDetail
{
Name = "Baz",
Version = "1.3.0",
RepoUri = GetRepoUrl(TestRepository.TestRepo1Name),
Commit = TestRepository.CoherencyTestRepo2Commit,
Type = DependencyType.Product,
Pinned = false,
CoherentParentDependencyName = "Foo"
};
DependencyDetail parentFoo = new DependencyDetail
{
Name = "Foo",
Version = "1.1.0",
RepoUri = GetRepoUrl(TestRepository.TestRepo2Name),
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
DependencyDetail parentBar = new DependencyDetail
{
Name = "Bar",
Version = "2.1.0",
RepoUri = GetRepoUrl(TestRepository.TestRepo2Name),
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedCoherencyDependencies.Add(parentFoo);
expectedCoherencyDependencies.Add(parentBar);
expectedCoherencyDependencies.Add(baz);
}
public ScenarioTests_AzDoFlow()
{
source1Assets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0");
source2Assets = GetAssetData("Pizza", "3.1.0", "Hamburger", "4.1.0");
source1AssetsUpdated = GetAssetData("Foo", "1.17.0", "Bar", "2.17.0");
expectedAzDoDependenciesSource1 = new List <DependencyDetail>();
string sourceRepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name);
DependencyDetail foo = new DependencyDetail
{
Name = "Foo",
Version = "1.1.0",
RepoUri = sourceRepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedAzDoDependenciesSource1.Add(foo);
DependencyDetail bar = new DependencyDetail
{
Name = "Bar",
Version = "2.1.0",
RepoUri = sourceRepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedAzDoDependenciesSource1.Add(bar);
expectedAzDoDependenciesSource2 = new List <DependencyDetail>();
string source2RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo3Name);
DependencyDetail pizza = new DependencyDetail
{
Name = "Pizza",
Version = "3.1.0",
RepoUri = source2RepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedAzDoDependenciesSource2.Add(pizza);
DependencyDetail hamburger = new DependencyDetail
{
Name = "Hamburger",
Version = "4.1.0",
RepoUri = source2RepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedAzDoDependenciesSource2.Add(hamburger);
expectedAzDoDependenciesSource1Updated = new List <DependencyDetail>();
DependencyDetail fooUpdated = new DependencyDetail
{
Name = "Foo",
Version = "1.1.0",
RepoUri = sourceRepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedAzDoDependenciesSource1Updated.Add(fooUpdated);
DependencyDetail barUpdated = new DependencyDetail
{
Name = "Bar",
Version = "2.1.0",
RepoUri = sourceRepoUri,
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false
};
expectedAzDoDependenciesSource1Updated.Add(barUpdated);
}
public async Task Darc_AzDoFlow_FeedFlow()
{
TestContext.WriteLine("AzDo Dependency Feed Flow, non-batched");
// Feed flow test strings
string proxyFeed = "https://some-proxy.azurewebsites.net/container/some-container/sig/somesig/se/2020-02-02/darc-int-maestro-test1-bababababab-1/index.json";
string azdoFeed1 = "https://some_org.pkgs.visualstudio.com/_packaging/darc-int-maestro-test1-aaabaababababe-1/nuget/v3/index.json";
string azdoFeed2 = "https://some_org.pkgs.visualstudio.com/_packaging/darc-int-maestro-test1-bbbbaababababd-1/nuget/v3/index.json";
string azdoFeed3 = "https://some_org.pkgs.visualstudio.com/_packaging/darc-int-maestro-test1-cccbaababababf-1/nuget/v3/index.json";
string regularFeed = "https://dotnetfeed.blob.core.windows.net/maestro-test1/index.json";
string buildContainer = "https://dev.azure.com/dnceng/internal/_apis/build/builds/9999999/artifacts";
string[] expectedFeeds = { proxyFeed, azdoFeed1, azdoFeed3 };
string[] notExpectedFeeds = { regularFeed, azdoFeed2, buildContainer };
IImmutableList <AssetData> feedFlowSourceAssets = ImmutableList.Create(
GetAssetDataWithLocations(
"Foo",
"1.1.0",
proxyFeed,
LocationType.NugetFeed
),
GetAssetDataWithLocations(
"Bar",
"2.1.0",
azdoFeed1,
LocationType.NugetFeed),
GetAssetDataWithLocations(
"Pizza",
"3.1.0",
azdoFeed2,
LocationType.NugetFeed,
regularFeed,
LocationType.NugetFeed
),
GetAssetDataWithLocations(
"Hamburger",
"4.1.0",
azdoFeed3,
LocationType.NugetFeed,
buildContainer,
LocationType.Container)
);
TestContext.WriteLine("Azure DevOps Internal feed flow");
TestParameters parameters = await TestParameters.GetAsync();
SetTestParameters(parameters);
EndToEndFlowLogic testLogic = new EndToEndFlowLogic(parameters);
List <DependencyDetail> expectedAzDoFeedFlowDependencies = new List <DependencyDetail>();
DependencyDetail feedFoo = new DependencyDetail
{
Name = "Foo",
Version = "1.1.0",
RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name),
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false,
Locations = new List <string> {
proxyFeed
}
};
expectedAzDoFeedFlowDependencies.Add(feedFoo);
DependencyDetail feedBar = new DependencyDetail
{
Name = "Bar",
Version = "2.1.0",
RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name),
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false,
Locations = new List <string> {
azdoFeed1
}
};
expectedAzDoFeedFlowDependencies.Add(feedBar);
DependencyDetail feedPizza = new DependencyDetail
{
Name = "Pizza",
Version = "3.1.0",
RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name),
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false,
Locations = new List <string> {
azdoFeed2, regularFeed
}
};
expectedAzDoFeedFlowDependencies.Add(feedPizza);
DependencyDetail feedHamburger = new DependencyDetail
{
Name = "Hamburger",
Version = "4.1.0",
RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name),
Commit = TestRepository.CoherencyTestRepo1Commit,
Type = DependencyType.Product,
Pinned = false,
Locations = new List <string> {
azdoFeed3, buildContainer
}
};
expectedAzDoFeedFlowDependencies.Add(feedHamburger);
await testLogic.NonBatchedAzDoFlowTestBase(
$"AzDo_FeedFlowBranch_{Environment.MachineName}",
$"AzDo_FeedFlowChannel_{Environment.MachineName}",
feedFlowSourceAssets,
expectedAzDoFeedFlowDependencies,
isFeedTest : true,
expectedFeeds : expectedFeeds,
notExpectedFeeds : notExpectedFeeds).ConfigureAwait(false);
}
private void LogDependency(DependencyDetail dependency)
{
Console.Write(UxHelpers.DependencyToString(dependency));
}
public async Task CoherencyUpdateTests10(bool pinHead)
{
// Initialize
Mock <IBarClient> barClientMock = new Mock <IBarClient>();
Remote remote = new Remote(null, barClientMock.Object, NullLogger.Instance);
// Mock the remote used by build dependency graph to gather dependency details.
Mock <IRemote> dependencyGraphRemoteMock = new Mock <IRemote>();
// Always return the main remote.
var remoteFactoryMock = new Mock <IRemoteFactory>();
remoteFactoryMock.Setup(m => m.GetRemoteAsync(It.IsAny <string>(), It.IsAny <ILogger>())).ReturnsAsync(dependencyGraphRemoteMock.Object);
remoteFactoryMock.Setup(m => m.GetBarOnlyRemoteAsync(It.IsAny <ILogger>())).ReturnsAsync(remote);
List <DependencyDetail> existingDetails = new List <DependencyDetail>();
DependencyDetail depA = AddDependency(existingDetails, "depA", "v1", "repoA", "commit1", pinned: pinHead);
DependencyDetail depB = AddDependency(existingDetails, "depB", "v3", "repoB", "commit2", pinned: false, coherentParent: "depA");
// Both C and D depend on B
DependencyDetail depC = AddDependency(existingDetails, "depC", "v0", "repoC", "commit3", pinned: false, coherentParent: "depB");
DependencyDetail depD = AddDependency(existingDetails, "depD", "v50", "repoD", "commit5", pinned: false, coherentParent: "depB");
BuildProducesAssets(barClientMock, "repoA", "commit1", new List <(string name, string version)>
{
("depA", "v1")
});
List <DependencyDetail> repoADeps = new List <DependencyDetail>();
AddDependency(repoADeps, "depY", "v42", "repoB", "commit5", pinned: false);
RepoHasDependencies(dependencyGraphRemoteMock, "repoA", "commit1", repoADeps);
BuildProducesAssets(barClientMock, "repoB", "commit5", new List <(string name, string version)>
{
("depB", "v10"),
("depY", "v42"),
});
List <DependencyDetail> repoBDeps = new List <DependencyDetail>();
AddDependency(repoBDeps, "depQ", "v66", "repoD", "commit35", pinned: false);
AddDependency(repoBDeps, "depZ", "v64", "repoC", "commit7", pinned: false);
RepoHasDependencies(dependencyGraphRemoteMock, "repoB", "commit5", repoBDeps);
BuildProducesAssets(barClientMock, "repoC", "commit7", new List <(string name, string version)>
{
("depC", "v1000"),
("depZ", "v64"),
});
BuildProducesAssets(barClientMock, "repoD", "commit35", new List <(string name, string version)>
{
("depD", "v1001"),
("depQ", "v66"),
});
// This should bring B and C in line.
List <DependencyUpdate> coherencyUpdates =
await remote.GetRequiredCoherencyUpdatesAsync(existingDetails, remoteFactoryMock.Object);
Assert.Collection(coherencyUpdates,
u =>
{
Assert.Equal(depC, u.From);
Assert.Equal("v1000", u.To.Version);
Assert.Equal("commit7", u.To.Commit);
Assert.Equal("repoC", u.To.RepoUri);
},
u =>
{
Assert.Equal(depB, u.From);
Assert.Equal("v10", u.To.Version);
Assert.Equal("commit5", u.To.Commit);
Assert.Equal("repoB", u.To.RepoUri);
},
u =>
{
Assert.Equal(depD, u.From);
Assert.Equal("v1001", u.To.Version);
Assert.Equal("commit35", u.To.Commit);
Assert.Equal("repoD", u.To.RepoUri);
});
}
private static async Task <int?> GetBuildId(DependencyDetail dep, IMaestroApi client, Dictionary <int, Client.Models.Build> buildCache,
Dictionary <(string name, string version, string commit), int> assetCache, CancellationToken cancellationToken)
/// <summary>
/// Update local dependencies based on a specific channel.
/// </summary>
/// <param name="options">Command line options</param>
/// <returns>Process exit code.</returns>
public override async Task <int> ExecuteAsync()
{
try
{
DarcSettings darcSettings = darcSettings = LocalSettings.GetDarcSettings(_options, Logger);
// TODO: PAT only used for pulling the arcade eng/common dir,
// so hardcoded to GitHub PAT right now. Must be more generic in the future.
darcSettings.GitType = GitRepoType.GitHub;
LocalSettings localSettings = LocalSettings.LoadSettingsFile(_options);
darcSettings.PersonalAccessToken = localSettings != null && !string.IsNullOrEmpty(localSettings.GitHubToken) ?
localSettings.GitHubToken :
_options.GitHubPat;
Remote remote = new Remote(darcSettings, Logger);
Local local = new Local(LocalHelpers.GetGitDir(Logger), Logger);
List <DependencyDetail> dependenciesToUpdate = new List <DependencyDetail>();
bool someUpToDate = false;
string finalMessage = $"Local dependencies updated from channel '{_options.Channel}'.";
// First we need to figure out what to query for. Load Version.Details.xml and
// find all repository uris, optionally restricted by the input dependency parameter.
IEnumerable <DependencyDetail> dependencies = await local.GetDependenciesAsync(_options.Name);
if (!dependencies.Any())
{
Console.WriteLine("Found no dependencies to update.");
return(Constants.ErrorCode);
}
if (!string.IsNullOrEmpty(_options.Name) && !string.IsNullOrEmpty(_options.Version))
{
DependencyDetail dependency = dependencies.First();
dependency.Version = _options.Version;
dependenciesToUpdate.Add(dependency);
Console.WriteLine($"Updating '{dependency.Name}': '{dependency.Version}' => '{_options.Version}'");
finalMessage = $"Local dependency {_options.Name} updated to version '{_options.Version}'.";
}
else if (!string.IsNullOrEmpty(_options.PackagesFolder))
{
try
{
dependenciesToUpdate.AddRange(GetDependenciesFromPackagesFolder(_options.PackagesFolder, dependencies));
}
catch (DarcException exc)
{
Logger.LogError(exc, $"Error: Failed to update dependencies based on folder '{_options.PackagesFolder}'");
return(Constants.ErrorCode);
}
finalMessage = $"Local dependencies updated based on packages folder {_options.PackagesFolder}.";
}
else
{
// Start channel query.
var channel = remote.GetChannelAsync(_options.Channel);
// Limit the number of BAR queries by grabbing the repo URIs and making a hash set.
var repositoryUrisForQuery = dependencies.Select(dependency => dependency.RepoUri).ToHashSet();
ConcurrentDictionary <string, Task <Build> > getLatestBuildTaskDictionary = new ConcurrentDictionary <string, Task <Build> >();
Channel channelInfo = await channel;
if (channelInfo == null)
{
Console.WriteLine($"Could not find a channel named '{_options.Channel}'.");
return(Constants.ErrorCode);
}
foreach (string repoToQuery in repositoryUrisForQuery)
{
var latestBuild = remote.GetLatestBuildAsync(repoToQuery, channelInfo.Id.Value);
getLatestBuildTaskDictionary.TryAdd(repoToQuery, latestBuild);
}
// Now walk dependencies again and attempt the update
foreach (DependencyDetail dependency in dependencies)
{
Build build;
try
{
build = await getLatestBuildTaskDictionary[dependency.RepoUri];
}
catch (ApiErrorException e) when(e.Response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
Logger.LogTrace($"No build of '{dependency.RepoUri}' found on channel '{_options.Channel}'.");
continue;
}
Asset buildAsset = build.Assets.Where(asset => asset.Name.Equals(dependency.Name, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (buildAsset == null)
{
Logger.LogTrace($"Dependency '{dependency.Name}' not found in latest build of '{dependency.RepoUri}' on '{_options.Channel}', skipping.");
continue;
}
if (buildAsset.Version == dependency.Version &&
buildAsset.Name == dependency.Name &&
build.Repository == dependency.RepoUri &&
build.Commit == dependency.Commit)
{
// No changes
someUpToDate = true;
continue;
}
DependencyDetail updatedDependency = new DependencyDetail
{
// TODO: Not needed, but not currently provided in Build info. Will be available on next rollout.
Branch = null,
Commit = build.Commit,
// If casing changes, ensure that the dependency name gets updated.
Name = buildAsset.Name,
RepoUri = build.Repository,
Version = buildAsset.Version
};
dependenciesToUpdate.Add(updatedDependency);
// Print out what we are going to do.
Console.WriteLine($"Updating '{dependency.Name}': '{dependency.Version}' => '{updatedDependency.Version}'" +
$" (from build '{build.BuildNumber}' of '{build.Repository}')");
// Notify on casing changes.
if (buildAsset.Name != dependency.Name)
{
Console.WriteLine($" Dependency name normalized to '{updatedDependency.Name}'");
}
dependenciesToUpdate.Add(updatedDependency);
}
}
if (!dependenciesToUpdate.Any())
{
// If we found some dependencies already up to date,
// then we consider this a success. Otherwise, we didn't even
// find matching dependencies so we should let the user know.
if (someUpToDate)
{
Console.WriteLine($"All dependencies are up to date.");
return(Constants.SuccessCode);
}
else
{
Console.WriteLine($"Found no dependencies to update.");
return(Constants.ErrorCode);
}
}
if (_options.DryRun)
{
return(Constants.SuccessCode);
}
// Now call the local updater to run the update
await local.UpdateDependenciesAsync(dependenciesToUpdate, remote);
Console.WriteLine(finalMessage);
return(Constants.SuccessCode);
}
catch (Exception e)
{
Logger.LogError(e, $"Error: Failed to update dependencies to channel {_options.Channel}");
return(Constants.ErrorCode);
}
}
/// <summary>
/// Update local dependencies based on a specific channel.
/// </summary>
/// <param name="options">Command line options</param>
/// <returns>Process exit code.</returns>
public override async Task <int> ExecuteAsync()
{
try
{
DarcSettings darcSettings = darcSettings = LocalSettings.GetDarcSettings(_options, Logger);
// TODO: PAT only used for pulling the Arcade eng/common dir,
// so hardcoded to GitHub PAT right now. Must be more generic in the future.
darcSettings.GitType = GitRepoType.GitHub;
LocalSettings localSettings = LocalSettings.LoadSettingsFile(_options);
darcSettings.GitRepoPersonalAccessToken = localSettings != null && !string.IsNullOrEmpty(localSettings.GitHubToken) ?
localSettings.GitHubToken :
_options.GitHubPat;
IRemoteFactory remoteFactory = new RemoteFactory(_options);
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(Logger);
Local local = new Local(Logger);
List <DependencyDetail> dependenciesToUpdate = new List <DependencyDetail>();
bool someUpToDate = false;
string finalMessage = $"Local dependencies updated from channel '{_options.Channel}'.";
// First we need to figure out what to query for. Load Version.Details.xml and
// find all repository uris, optionally restricted by the input dependency parameter.
IEnumerable <DependencyDetail> localDependencies = await local.GetDependenciesAsync(_options.Name, false);
// If the source repository was specified, filter away any local dependencies not from that
// source repository.
if (!string.IsNullOrEmpty(_options.SourceRepository))
{
localDependencies = localDependencies.Where(
dependency => dependency.RepoUri.Contains(_options.SourceRepository, StringComparison.OrdinalIgnoreCase));
}
if (!localDependencies.Any())
{
Console.WriteLine("Found no dependencies to update.");
return(Constants.ErrorCode);
}
List <DependencyDetail> currentDependencies = localDependencies.ToList();
if (!string.IsNullOrEmpty(_options.Name) && !string.IsNullOrEmpty(_options.Version))
{
DependencyDetail dependency = currentDependencies.First();
dependency.Version = _options.Version;
dependenciesToUpdate.Add(dependency);
Console.WriteLine($"Updating '{dependency.Name}': '{dependency.Version}' => '{_options.Version}'");
finalMessage = $"Local dependency {_options.Name} updated to version '{_options.Version}'.";
}
else if (!string.IsNullOrEmpty(_options.PackagesFolder))
{
try
{
dependenciesToUpdate.AddRange(GetDependenciesFromPackagesFolder(_options.PackagesFolder, currentDependencies));
}
catch (DarcException exc)
{
Logger.LogError(exc, $"Error: Failed to update dependencies based on folder '{_options.PackagesFolder}'");
return(Constants.ErrorCode);
}
finalMessage = $"Local dependencies updated based on packages folder {_options.PackagesFolder}.";
}
else if (_options.BARBuildId > 0)
{
try
{
if (!_options.CoherencyOnly)
{
Console.WriteLine($"Looking up build with BAR id {_options.BARBuildId}");
var specificBuild = await barOnlyRemote.GetBuildAsync(_options.BARBuildId);
int nonCoherencyResult = await NonCoherencyUpdatesForBuildAsync(specificBuild, barOnlyRemote, currentDependencies, dependenciesToUpdate)
.ConfigureAwait(false);
if (nonCoherencyResult != Constants.SuccessCode)
{
Console.WriteLine("Error: Failed to update non-coherent parent tied dependencies.");
return(nonCoherencyResult);
}
string sourceRepo = specificBuild.GitHubRepository ?? specificBuild.AzureDevOpsRepository;
string sourceBranch = specificBuild.GitHubBranch ?? specificBuild.AzureDevOpsBranch;
finalMessage = $"Local dependencies updated based on build with BAR id {_options.BARBuildId} " +
$"({specificBuild.AzureDevOpsBuildNumber} from {sourceRepo}@{sourceBranch})";
}
int coherencyResult = await CoherencyUpdatesAsync(barOnlyRemote, remoteFactory, currentDependencies, dependenciesToUpdate)
.ConfigureAwait(false);
if (coherencyResult != Constants.SuccessCode)
{
Console.WriteLine("Error: Failed to update coherent parent tied dependencies.");
return(coherencyResult);
}
finalMessage = string.IsNullOrEmpty(finalMessage) ? "Local dependencies successfully updated." : finalMessage;
}
catch (RestApiException e) when(e.Response.Status == 404)
{
Console.WriteLine($"Could not find build with BAR id '{_options.BARBuildId}'.");
return(Constants.ErrorCode);
}
}
else
{
if (!_options.CoherencyOnly)
{
if (string.IsNullOrEmpty(_options.Channel))
{
Console.WriteLine($"Please supply either a channel name (--channel), a packages folder (--packages-folder) " +
"a BAR build id (--id), or a specific dependency name and version (--name and --version).");
return(Constants.ErrorCode);
}
// Start channel query.
Task <Channel> channel = barOnlyRemote.GetChannelAsync(_options.Channel);
// Limit the number of BAR queries by grabbing the repo URIs and making a hash set.
// We gather the latest build for any dependencies that aren't marked with coherent parent
// dependencies, as those will be updated based on additional queries.
HashSet <string> repositoryUrisForQuery = currentDependencies
.Where(dependency => string.IsNullOrEmpty(dependency.CoherentParentDependencyName))
.Select(dependency => dependency.RepoUri)
.ToHashSet();
ConcurrentDictionary <string, Task <Build> > getLatestBuildTaskDictionary = new ConcurrentDictionary <string, Task <Build> >();
Channel channelInfo = await channel;
if (channelInfo == null)
{
Console.WriteLine($"Could not find a channel named '{_options.Channel}'.");
return(Constants.ErrorCode);
}
foreach (string repoToQuery in repositoryUrisForQuery)
{
Console.WriteLine($"Looking up latest build of {repoToQuery} on {_options.Channel}");
var latestBuild = barOnlyRemote.GetLatestBuildAsync(repoToQuery, channelInfo.Id);
getLatestBuildTaskDictionary.TryAdd(repoToQuery, latestBuild);
}
// For each build, first go through and determine the required updates,
// updating the "live" dependency information as we go.
// Then run a second pass where we update any assets based on coherency information.
foreach (KeyValuePair <string, Task <Build> > buildKvPair in getLatestBuildTaskDictionary)
{
string repoUri = buildKvPair.Key;
Build build = await buildKvPair.Value;
if (build == null)
{
Logger.LogTrace($"No build of '{repoUri}' found on channel '{_options.Channel}'.");
continue;
}
int nonCoherencyResult = await NonCoherencyUpdatesForBuildAsync(build, barOnlyRemote, currentDependencies, dependenciesToUpdate)
.ConfigureAwait(false);
if (nonCoherencyResult != Constants.SuccessCode)
{
Console.WriteLine("Error: Failed to update non-coherent parent tied dependencies.");
return(nonCoherencyResult);
}
}
}
int coherencyResult = await CoherencyUpdatesAsync(barOnlyRemote, remoteFactory, currentDependencies, dependenciesToUpdate)
.ConfigureAwait(false);
if (coherencyResult != Constants.SuccessCode)
{
Console.WriteLine("Error: Failed to update coherent parent tied dependencies.");
return(coherencyResult);
}
}
if (!dependenciesToUpdate.Any())
{
// If we found some dependencies already up to date,
// then we consider this a success. Otherwise, we didn't even
// find matching dependencies so we should let the user know.
if (someUpToDate)
{
Console.WriteLine($"All dependencies are up to date.");
return(Constants.SuccessCode);
}
else
{
Console.WriteLine($"Found no dependencies to update.");
return(Constants.ErrorCode);
}
}
if (_options.DryRun)
{
return(Constants.SuccessCode);
}
// Now call the local updater to run the update.
await local.UpdateDependenciesAsync(dependenciesToUpdate, remoteFactory);
Console.WriteLine(finalMessage);
return(Constants.SuccessCode);
}
catch (AuthenticationException e)
{
Console.WriteLine(e.Message);
return(Constants.ErrorCode);
}
catch (Exception e)
{
Logger.LogError(e, "Error: Failed to update dependencies.");
return(Constants.ErrorCode);
}
}
/// <summary>
/// Update local dependencies based on a specific channel.
/// </summary>
/// <param name="options">Command line options</param>
/// <returns>Process exit code.</returns>
public override async Task <int> ExecuteAsync()
{
try
{
DarcSettings darcSettings = darcSettings = LocalSettings.GetDarcSettings(_options, Logger);
// TODO: PAT only used for pulling the arcade eng/common dir,
// so hardcoded to GitHub PAT right now. Must be more generic in the future.
darcSettings.GitType = GitRepoType.GitHub;
LocalSettings localSettings = LocalSettings.LoadSettingsFile(_options);
darcSettings.GitRepoPersonalAccessToken = localSettings != null && !string.IsNullOrEmpty(localSettings.GitHubToken) ?
localSettings.GitHubToken :
_options.GitHubPat;
IRemoteFactory remoteFactory = new RemoteFactory(_options);
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(Logger);
Local local = new Local(Logger);
List <DependencyDetail> dependenciesToUpdate = new List <DependencyDetail>();
bool someUpToDate = false;
string finalMessage = $"Local dependencies updated from channel '{_options.Channel}'.";
// First we need to figure out what to query for. Load Version.Details.xml and
// find all repository uris, optionally restricted by the input dependency parameter.
IEnumerable <DependencyDetail> localDependencies = await local.GetDependenciesAsync(_options.Name, false);
// If the source repository was specified, filter away any local dependencies not from that
// source repository.
if (!string.IsNullOrEmpty(_options.SourceRepository))
{
localDependencies = localDependencies.Where(
dependency => dependency.RepoUri.Contains(_options.SourceRepository, StringComparison.OrdinalIgnoreCase));
}
if (!localDependencies.Any())
{
Console.WriteLine("Found no dependencies to update.");
return(Constants.ErrorCode);
}
List <DependencyDetail> currentDependencies = localDependencies.ToList();
if (!string.IsNullOrEmpty(_options.Name) && !string.IsNullOrEmpty(_options.Version))
{
DependencyDetail dependency = currentDependencies.First();
dependency.Version = _options.Version;
dependenciesToUpdate.Add(dependency);
Console.WriteLine($"Updating '{dependency.Name}': '{dependency.Version}' => '{_options.Version}'");
finalMessage = $"Local dependency {_options.Name} updated to version '{_options.Version}'.";
}
else if (!string.IsNullOrEmpty(_options.PackagesFolder))
{
try
{
dependenciesToUpdate.AddRange(GetDependenciesFromPackagesFolder(_options.PackagesFolder, currentDependencies));
}
catch (DarcException exc)
{
Logger.LogError(exc, $"Error: Failed to update dependencies based on folder '{_options.PackagesFolder}'");
return(Constants.ErrorCode);
}
finalMessage = $"Local dependencies updated based on packages folder {_options.PackagesFolder}.";
}
else
{
if (!_options.CoherencyOnly)
{
if (string.IsNullOrEmpty(_options.Channel))
{
Console.WriteLine($"Please supply either a channel name (--channel), a packages folder (--packages-folder) " +
$"or a specific dependency name and version (--name and --version).");
return(Constants.ErrorCode);
}
// Start channel query.
Task <Channel> channel = barOnlyRemote.GetChannelAsync(_options.Channel);
// Limit the number of BAR queries by grabbing the repo URIs and making a hash set.
// We gather the latest build for any dependencies that aren't marked with coherent parent
// dependencies, as those will be updated based on additional queries.
HashSet <string> repositoryUrisForQuery = currentDependencies
.Where(dependency => string.IsNullOrEmpty(dependency.CoherentParentDependencyName))
.Select(dependency => dependency.RepoUri)
.ToHashSet();
ConcurrentDictionary <string, Task <Build> > getLatestBuildTaskDictionary = new ConcurrentDictionary <string, Task <Build> >();
Channel channelInfo = await channel;
if (channelInfo == null)
{
Console.WriteLine($"Could not find a channel named '{_options.Channel}'.");
return(Constants.ErrorCode);
}
foreach (string repoToQuery in repositoryUrisForQuery)
{
Console.WriteLine($"Looking up latest build of {repoToQuery} on {_options.Channel}");
var latestBuild = barOnlyRemote.GetLatestBuildAsync(repoToQuery, channelInfo.Id);
getLatestBuildTaskDictionary.TryAdd(repoToQuery, latestBuild);
}
// For each build, first go through and determine the required updates,
// updating the "live" dependency information as we go.
// Then run a second pass where we update any assets based on coherency information.
foreach (KeyValuePair <string, Task <Build> > buildKvPair in getLatestBuildTaskDictionary)
{
string repoUri = buildKvPair.Key;
Build build = await buildKvPair.Value;
if (build == null)
{
Logger.LogTrace($"No build of '{repoUri}' found on channel '{_options.Channel}'.");
continue;
}
IEnumerable <AssetData> assetData = build.Assets.Select(
a => new AssetData(a.NonShipping)
{
Name = a.Name,
Version = a.Version
});
// Now determine what needs to be updated.
List <DependencyUpdate> updates = await barOnlyRemote.GetRequiredNonCoherencyUpdatesAsync(
repoUri, build.Commit, assetData, currentDependencies);
foreach (DependencyUpdate update in updates)
{
DependencyDetail from = update.From;
DependencyDetail to = update.To;
// Print out what we are going to do.
Console.WriteLine($"Updating '{from.Name}': '{from.Version}' => '{to.Version}'" +
$" (from build '{build.AzureDevOpsBuildNumber}' of '{repoUri}')");
// Final list of dependencies to update
dependenciesToUpdate.Add(to);
// Replace in the current dependencies list so the correct data is fed into the coherency pass.
currentDependencies.Remove(from);
currentDependencies.Add(to);
}
}
}
Console.WriteLine("Checking for coherency updates...");
// Now run a coherency update based on the current set of dependencies updated
// from the previous pass.
List <DependencyUpdate> coherencyUpdates =
await barOnlyRemote.GetRequiredCoherencyUpdatesAsync(currentDependencies, remoteFactory);
foreach (DependencyUpdate dependencyUpdate in coherencyUpdates)
{
DependencyDetail from = dependencyUpdate.From;
DependencyDetail to = dependencyUpdate.To;
DependencyDetail coherencyParent = currentDependencies.First(d =>
d.Name.Equals(from.CoherentParentDependencyName, StringComparison.OrdinalIgnoreCase));
// Print out what we are going to do.
Console.WriteLine($"Updating '{from.Name}': '{from.Version}' => '{to.Version}' " +
$"to ensure coherency with {from.CoherentParentDependencyName}@{coherencyParent.Version}");
// Final list of dependencies to update
dependenciesToUpdate.Add(to);
}
}
if (!dependenciesToUpdate.Any())
{
// If we found some dependencies already up to date,
// then we consider this a success. Otherwise, we didn't even
// find matching dependencies so we should let the user know.
if (someUpToDate)
{
Console.WriteLine($"All dependencies are up to date.");
return(Constants.SuccessCode);
}
else
{
Console.WriteLine($"Found no dependencies to update.");
return(Constants.ErrorCode);
}
}
if (_options.DryRun)
{
return(Constants.SuccessCode);
}
// Now call the local updater to run the update.
await local.UpdateDependenciesAsync(dependenciesToUpdate, remoteFactory);
Console.WriteLine(finalMessage);
return(Constants.SuccessCode);
}
catch (Exception e)
{
Logger.LogError(e, $"Error: Failed to update dependencies to channel {_options.Channel}");
return(Constants.ErrorCode);
}
}
