public async Task CreatesNoopRunnerWhenNoPoliciesFound()
{
//arrange
var repositoryId = "repo-id";
var extensionId = "ext-id";
var publisherId = "pub-id";
var organization = "org";
ServiceProvider serviceProvider = CreateServiceProvider(extensionId, publisherId);
HttpClient httpClient = CreateHttpMockWith404Response(repositoryId, extensionId, publisherId, organization);
var settings = serviceProvider.GetService <IOptionsMonitor <ExtensionSettings> >();
var azDoClient = new AzureDevOpsClient(httpClient, "token", settings);
var subject = new MergePolicyRunnerFactory(
serviceProvider,
serviceProvider.GetService <ILogger <MergePolicyRunnerFactory> >()
);
var context = new MergePolicyRunnerFactoryContext(azDoClient, repositoryId, organization);
//act
var result = await subject.CreateAsync(context);
//assert
Assert.IsType <NoopMergePolicyRunner>(result);
}
public async Task Test()
{
var zipExtractor = new ZipExtractor(new FileSystemOperations());
var azureDevOpsClient = AzureDevOpsClient.Create(new Uri("https://dev.azure.com"));
var task = new AzureDevOpsUnpack("LumiaWOA;Lumia950XLPkg;1;MSM8994 UEFI (Lumia 950 XL)", azureDevOpsClient, zipExtractor);
await task.Execute();
}
public async Task CreatesNewRunnerWhenConfigurationChanges()
{
//arrange
var repositoryId = "repo-id";
var extensionId = "ext-id";
var publisherId = "pub-id";
var organization = "org";
ServiceProvider serviceProvider = CreateServiceProvider(extensionId, publisherId);
HttpClient httpClient = CreateHttpMock(repositoryId, extensionId, publisherId, organization);
var settings = serviceProvider.GetService <IOptionsMonitor <ExtensionSettings> >();
var azDoClient = new AzureDevOpsClient(httpClient, "token", settings);
var subject = new MergePolicyRunnerFactory(
serviceProvider,
serviceProvider.GetService <ILogger <MergePolicyRunnerFactory> >()
);
var context = new MergePolicyRunnerFactoryContext(azDoClient, repositoryId, organization);
//act
var result1 = await subject.CreateAsync(context);
subject.Clear(new MergePolicyRunnerFactoryContext(context.RepositoryId, context.Organization));
var result2 = await subject.CreateAsync(context);
//assert
Assert.IsType <MergePolicyRunner>(result1);
Assert.IsType <MergePolicyRunner>(result2);
Assert.NotSame(result1, result2);
}
public static IExportRegistrationBlock WithCommon(this IExportRegistrationBlock block,
WindowsDeploymentOptionsProvider installOptionsProvider)
{
var taskTypes = from a in Assemblies.AppDomainAssemblies
from type in a.ExportedTypes
where type.GetTypeInfo().ImplementedInterfaces.Contains(typeof(IDeploymentTask))
select type;
block.ExportAssemblies(Assemblies.AppDomainAssemblies).ByInterface <ISpaceAllocator <IDevice> >();
block.Export <ZipExtractor>().As <IZipExtractor>();
block.ExportFactory(Tokenizer.Create).As <Tokenizer <LangToken> >();
block.Export <ScriptParser>().As <IScriptParser>();
block.ExportFactory(() => installOptionsProvider).As <IWindowsOptionsProvider>();
block.Export <WoaDeployer>().As <IWoaDeployer>();
block.Export <BootCreator>().As <IBootCreator>();
block.Export <LowLevelApi>().As <ILowLevelApi>();
block.ExportInstance(taskTypes).As <IEnumerable <Type> >();
block.Export <ScriptRunner>().As <IScriptRunner>();
block.Export <InstanceBuilder>().As <IInstanceBuilder>();
block.Export <RaspberryPathBuilder>().As <IPathBuilder>();
block.Export <FileSystemOperations>().As <IFileSystemOperations>();
block.Export <BcdInvokerFactory>().As <IBcdInvokerFactory>();
block.Export <WindowsDeployer>().As <IWindowsDeployer>();
block.Export <RaspberryDisklayoutPreparer>().As <IDiskLayoutPreparer>();
block.Export <ImageFlasher>().As <IImageFlasher>();
block.Export <DismImageService>().As <IWindowsImageService>();
block.ExportFactory(() => AzureDevOpsClient.Create(new Uri("https://dev.azure.com"))).As <IAzureDevOpsBuildClient>();
return(block);
}
public static async Task Main()
{
var options = new AzureDevOpsClientOptions();
var workItemIds = new HashSet <int>();
#if DEBUG
if (TryReadSettings(out var settings))
{
options.Organization = settings.GetValueOrDefault("Organization", () => string.Empty);
options.PersonalAccessToken = settings.GetValueOrDefault("PersonalAccessToken", () => string.Empty);
options.Project = settings.GetValueOrDefault("Project", () => string.Empty);
workItemIds = settings.GetValueOrDefault("WorkItemIds", () => string.Empty)
.Split(new[] { ' ', ',', ';' }, StringSplitOptions.RemoveEmptyEntries)
.Select(int.Parse)
.ToHashSet();
}
#endif
var includeWorkItemTypes = new HashSet <WorkItemType>
{
WorkItemType.Epic,
WorkItemType.Feature,
WorkItemType.PBI
};
var client = new AzureDevOpsClient(options);
var crawler = new DirectLinksCrawler(client);
var data = await crawler.GetData(workItemIds, includeWorkItemTypes, includeFinishedWorkItems : false);
var graph = new GraphGenerator().GenerateGraph(data, workItemIds, "LR");
// http://magjac.com/graphviz-visual-editor/ can be used to test the graph
System.Console.WriteLine(graph);
}
public static async Task <string> GetCommitAuthorAsync(
string repositoryUrl,
string commit,
string gitHubPersonalAccessToken,
string azureDevOpsPersonalAccessToken)
{
if (Uri.TryCreate(repositoryUrl, UriKind.Absolute, out Uri parsedUri))
{
if (parsedUri.Host == "github.com")
{
if (string.IsNullOrEmpty(gitHubPersonalAccessToken))
{
throw new ArgumentException("A GitHub personal access token is needed for this operation.");
}
return(await GitHubClient.GetCommitAuthorAsync(repositoryUrl, commit, gitHubPersonalAccessToken));
}
if (string.IsNullOrEmpty(azureDevOpsPersonalAccessToken))
{
throw new ArgumentException("An Azure DevOps personal access token is needed for this operation.");
}
return(await AzureDevOpsClient.GetCommitAuthorAsync(repositoryUrl, commit, azureDevOpsPersonalAccessToken));
}
throw new InvalidCastException($"'{parsedUri}' is not a valid URI");
}
public void ParseValidRepoUriTests(string inputUri, string expectedAccount, string expectedProject, string expectedRepo)
{
(string account, string project, string repo) = AzureDevOpsClient.ParseRepoUri(inputUri);
account.Should().Be(expectedAccount);
project.Should().Be(expectedProject);
repo.Should().Be(expectedRepo);
}
public static async Task Main(string[] args)
{
// Parse command line
CommandOptions parsedOptions = Program.ParseArguments(args);
if (parsedOptions == null)
{
return;
}
// Connect to Azure DevOps client
AzureDevOpsClient client = new AzureDevOpsClient(parsedOptions.Organization);
await client.ConnectAsync();
// Display token information
Console.WriteLine($"Organization: {client.VssConnection.Uri}");
Console.WriteLine($"AccessToken: {client.AuthenticationResult.AccessToken}");
Console.WriteLine($"AccessTokenType: {client.AuthenticationResult.AccessTokenType}");
Console.WriteLine($"ExpiresOn: {client.AuthenticationResult.ExpiresOn}");
Console.WriteLine($"TenantId: {client.AuthenticationResult.TenantId}");
Console.WriteLine($"GivenName: {client.AuthenticationResult.UserInfo?.GivenName}");
Console.WriteLine($"FamilyName: {client.AuthenticationResult.UserInfo?.FamilyName}");
Console.WriteLine($"Authority: {client.AuthenticationResult.Authority}");
// Get list of projects and display
ProjectHttpClient projectHttpClient = await client.VssConnection.GetClientAsync <ProjectHttpClient>();
IPagedList <TeamProjectReference> projects = await projectHttpClient.GetProjects(top : 500);
Console.Write("Projects: ");
foreach (TeamProjectReference project in projects)
{
Console.Write($"{project.Name} ");
}
}
private void ParseValidRepoUriTests(string inputUri, string expectedAccount, string expectedProject, string expectedRepo)
{
(string account, string project, string repo) = AzureDevOpsClient.ParseRepoUri(inputUri);
Xunit.Assert.Equal(expectedAccount, account);
Xunit.Assert.Equal(expectedProject, project);
Xunit.Assert.Equal(expectedRepo, repo);
}
private async void OnLoaded(object sender, RoutedEventArgs args)
{
this.cancellationTokenSource = new CancellationTokenSource();
try
{
AzureDevOpsUserContext devopsUserContext;
using (this.ViewModel.Window.ProgressBar.Begin(this.cancellationTokenSource.Cancel, DevOps.Plugin.Resources.LoginPage_ProgressText))
{
devopsUserContext = await AzureDevOpsClient.GetUserContext();
}
PullRequestPageVM vm = new PullRequestPageVM(this.Tab, devopsUserContext);
this.Tab.ViewElement = new PullRequestPage(vm);
}
catch (Exception ex)
{
this.ViewModel.DisplayText = DevOps.Plugin.Resources.LoginPage_ErrorText;
this.ViewModel.InfoText = ex.ToString();
}
finally
{
this.cancellationTokenSource.Cancel();
this.cancellationTokenSource.Dispose();
this.cancellationTokenSource = null;
}
}
public void Dispose()
{
this.accountClient?.Dispose();
this.accountClient = null;
this.avatarHttpClient.CancelPendingRequests();
this.avatarHttpClient.Dispose();
}
private void ParseValidPullRequestUriTests(string inputUri, string expectedAccount,
string expectedProject, string expectedRepo, int expectedId)
{
(string account, string project, string repo, int id) = AzureDevOpsClient.ParsePullRequestUri(inputUri);
Xunit.Assert.Equal(expectedAccount, account);
Xunit.Assert.Equal(expectedProject, project);
Xunit.Assert.Equal(expectedRepo, repo);
Xunit.Assert.Equal(expectedId, id);
}
public void ParseValidPullRequestUriTests(string inputUri, string expectedAccount,
string expectedProject, string expectedRepo, int expectedId)
{
(string account, string project, string repo, int id) = AzureDevOpsClient.ParsePullRequestUri(inputUri);
account.Should().Be(expectedAccount);
project.Should().Be(expectedProject);
repo.Should().Be(expectedRepo);
id.Should().Be(expectedId);
}
public DependencyUpdater(IAzureDevOpsConfig config)
{
var client = new AzureDevOpsClient(config);
_pullRequestCreator = new PullRequestCreator(client);
_fileFetcher = new CSharpProjectFileFetcher(client);
_updateChecker = new DependencyUpdateChecker();
_fileUpdater = new CSharpProjectFileUpdater();
_fileParser = new CSharpProjectParser();
}
public WikiController()
{
pat = Env.GetEnvironmentVariable(Env.Keys.PAT);
accountName = Env.GetEnvironmentVariable(Env.Keys.ORGANIZATION);
projectId = Guid.Parse(Env.GetEnvironmentVariable(Env.Keys.PROJECTID));
repositoryId = Guid.Parse(Env.GetEnvironmentVariable(Env.Keys.WIKIID));
markdownServerUrl = Env.GetEnvironmentVariable(Env.Keys.MARKDOWNSERVICE);
sharedAccessSignature = Env.GetEnvironmentVariable(Env.Keys.SAS);
client = new AzureDevOpsClient(accountName, pat);
}
public async Task Test()
{
var zipExtractor = new ZipExtractor(new FileSystemOperations());
var azureDevOpsClient = AzureDevOpsClient.Create(new Uri("https://dev.azure.com"));
using (var httpClient = new HttpClient())
{
IDownloader downloader = new Downloader(httpClient);
var task = new FetchAzureDevOpsArtifact("LumiaWOA;Lumia950XLPkg;1;MSM8994 UEFI (Lumia 950 XL)", azureDevOpsClient, zipExtractor, downloader, null);
await task.Execute();
}
}
public static async Task <SkillResponse> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequestMessage httpRequestMessage, ILogger log)
{
log.LogTrace($"Controller Started");
WorkItemStore workItemStore;
try
{
// Loading Environment Information
string azureDevopsHostname = Environment.GetEnvironmentVariable("AzureDevOps:Hostname", EnvironmentVariableTarget.Process);
string azureDevopsUsername = string.Empty;
string azureDevopsAccessToken = Environment.GetEnvironmentVariable("AzureDevOps:PAT", EnvironmentVariableTarget.Process);
// Load Clients
AzureDevOpsClient azureDevOpsClient = new AzureDevOpsClient(new Uri(azureDevopsHostname), azureDevopsUsername, azureDevopsAccessToken);
workItemStore = await azureDevOpsClient.FetchWorkItemStoreAsync()
.ConfigureAwait(false);
}
catch (Exception ex)
{
log.LogError($"Failed to fetch work item store: {ex.Message}", ex);
throw;
}
// try to read request
SkillRequest skillRequest;
try
{
skillRequest = await httpRequestMessage.Content.ReadAsAsync <SkillRequest>()
.ConfigureAwait(false);
}
catch (Exception ex)
{
log.LogError($"Failed to read skill from request message: {ex.Message}", ex);
throw;
}
if (AzDoBridgeRequestFactory.TryGetRequest(skillRequest.Request.Type, log, out IAzDoBridgeRequest request))
{
return(await request.Handle(workItemStore, skillRequest).ConfigureAwait(false));
}
log.LogTrace($"No Speech Detected");
return(ResponseBuilder.Tell("No Speech Detected"));
}
public async Task <IRemote> GetRemoteAsync(string repoUrl, ILogger logger)
{
using (logger.BeginScope($"Getting remote for repo {repoUrl}."))
{
// Normalize the url with the AzDO client prior to attempting to
// get a token. When we do coherency updates we build a repo graph and
// may end up traversing links to classic azdo uris.
string normalizedUrl = AzureDevOpsClient.NormalizeUrl(repoUrl);
Uri normalizedRepoUri = new Uri(normalizedUrl);
// Look up the setting for where the repo root should be held. Default to empty,
// which will use the temp directory.
string temporaryRepositoryRoot = Configuration.GetValue <string>("DarcTemporaryRepoRoot", null);
if (string.IsNullOrEmpty(temporaryRepositoryRoot))
{
temporaryRepositoryRoot = _tempFiles.GetFilePath("repos");
}
IGitRepo gitClient;
long installationId = await Context.GetInstallationId(normalizedUrl);
await ExponentialRetry.RetryAsync(
async() => await EnsureLocalGit(logger),
ex => logger.LogError(ex, $"Failed to install git to local temporary directory."),
ex => true);
switch (normalizedRepoUri.Host)
{
case "github.com":
if (installationId == default)
{
throw new GithubApplicationInstallationException($"No installation is avaliable for repository '{normalizedUrl}'");
}
gitClient = new GitHubClient(_gitExecutable, await GitHubTokenProvider.GetTokenForInstallationAsync(installationId),
logger, temporaryRepositoryRoot, Cache.Cache);
break;
case "dev.azure.com":
gitClient = new AzureDevOpsClient(_gitExecutable, await AzureDevOpsTokenProvider.GetTokenForRepository(normalizedUrl),
logger, temporaryRepositoryRoot);
break;
default:
throw new NotImplementedException($"Unknown repo url type {normalizedUrl}");
}
;
return(new Remote(gitClient, new MaestroBarClient(Context), logger));
}
}
/// <summary>
/// Get a remote for a specific repo.
/// </summary>
/// <param name="options">Command line options</param>
/// <param name="repoUrl">Repository url</param>
/// <param name="logger">Logger</param>
/// <returns>New remote</returns>
public static IRemote GetRemote(CommandLineOptions options, string repoUrl, ILogger logger)
{
DarcSettings darcSettings = LocalSettings.GetDarcSettings(options, logger, repoUrl);
if (darcSettings.GitType != GitRepoType.None &&
string.IsNullOrEmpty(darcSettings.GitRepoPersonalAccessToken))
{
throw new DarcException($"No personal access token was provided for repo type '{darcSettings.GitType}'");
}
// If a temporary repository root was not provided, use the environment
// provided temp directory.
string temporaryRepositoryRoot = darcSettings.TemporaryRepositoryRoot;
if (string.IsNullOrEmpty(temporaryRepositoryRoot))
{
temporaryRepositoryRoot = Path.GetTempPath();
}
IGitRepo gitClient = null;
if (darcSettings.GitType == GitRepoType.GitHub)
{
gitClient = new GitHubClient(options.GitLocation, darcSettings.GitRepoPersonalAccessToken,
logger,
temporaryRepositoryRoot,
// Caching not in use for Darc local client.
null);
}
else if (darcSettings.GitType == GitRepoType.AzureDevOps)
{
gitClient = new AzureDevOpsClient(options.GitLocation, darcSettings.GitRepoPersonalAccessToken,
logger,
temporaryRepositoryRoot);
}
IBarClient barClient = null;
if (!string.IsNullOrEmpty(darcSettings.BuildAssetRegistryPassword))
{
barClient = new MaestroApiBarClient(darcSettings.BuildAssetRegistryPassword,
darcSettings.BuildAssetRegistryBaseUri);
}
return(new Remote(gitClient, barClient, logger));
}
public async Task OwaspZapFailures_ShouldCreateBugs()
{
string collectionUri = "https://myaccount.visualstudio.com";
string teamProjectName = "My Team Project";
string team = "SRE";
string releaseUri = "vstfs:///ReleaseManagement/Release/209";
string releaseEnvironmentUri = "vstfs:///ReleaseManagement/Environment/538";
string filePath = @"C:\alerts\full_OwaspZapAlerts.xml";
bool failOnHigh = false;
string personalAccessToken = "1234";
string targetUrl = "https://mysite.azurewebsites.net";
string bugTitlePrefix = "Unit Test";
var client = new AzureDevOpsClient();
await client.CreateBugFromPenTestAsync(collectionUri, teamProjectName, team, releaseUri, releaseEnvironmentUri, filePath, failOnHigh, personalAccessToken, targetUrl, bugTitlePrefix);
}
public async Task <IRemote> GetRemoteAsync(string repoUrl, ILogger logger)
{
// Normalize the url with the AzDO client prior to attempting to
// get a token. When we do coherency updates we build a repo graph and
// may end up traversing links to classic azdo uris.
string normalizedUrl = AzureDevOpsClient.NormalizeUrl(repoUrl);
Uri normalizedRepoUri = new Uri(normalizedUrl);
// Look up the setting for where the repo root should be held. Default to empty,
// which will use the temp directory.
string temporaryRepositoryRoot = Configuration.GetValue <string>("DarcTemporaryRepoRoot", null);
if (string.IsNullOrEmpty(temporaryRepositoryRoot))
{
temporaryRepositoryRoot = Path.GetTempPath();
}
IGitRepo gitClient;
long installationId = await Context.GetInstallationId(normalizedUrl);
switch (normalizedRepoUri.Host)
{
case "github.com":
if (installationId == default)
{
throw new SubscriptionException($"No installation is avaliable for repository '{normalizedUrl}'");
}
gitClient = new GitHubClient(await GitHubTokenProvider.GetTokenForInstallation(installationId),
logger, temporaryRepositoryRoot);
break;
case "dev.azure.com":
gitClient = new AzureDevOpsClient(await AzureDevOpsTokenProvider.GetTokenForRepository(normalizedUrl),
logger, temporaryRepositoryRoot);
break;
default:
throw new NotImplementedException($"Unknown repo url type {normalizedUrl}");
}
;
return(new Remote(gitClient, new MaestroBarClient(Context), logger));
}
/// <summary>
/// Given a downloaded build, determine what the product name should be
/// for the release json
/// </summary>
/// <param name="build">Downloaded build</param>
/// <returns>Product name</returns>
private string GetProductNameForReleaseJson(DownloadedBuild build)
{
// Preference the github repo name over the azure devops repo name.
if (!string.IsNullOrEmpty(build.Build.GitHubRepository))
{
// Split off the github.com+org name and just use the repo name, all lower case.
(string owner, string repo) = GitHubClient.ParseRepoUri(build.Build.GitHubRepository);
return(repo.ToLowerInvariant());
}
else if (!string.IsNullOrEmpty(build.Build.AzureDevOpsRepository))
{
// Use the full repo name without project/account
(string accountName, string projectName, string repoName) = AzureDevOpsClient.ParseRepoUri(build.Build.AzureDevOpsRepository);
return(repoName.ToLowerInvariant());
}
else
{
throw new NotImplementedException("Unknown repository name.");
}
}
public static IExportRegistrationBlock WithCommon(this IExportRegistrationBlock block,
WindowsDeploymentOptionsProvider installOptionsProvider)
{
var taskTypes = from a in Assemblies.AppDomainAssemblies
from type in a.ExportedTypes
where type.GetTypeInfo().ImplementedInterfaces.Contains(typeof(IDeploymentTask))
select type;
block.ExportAssemblies(Assemblies.AppDomainAssemblies).ByInterface <ISpaceAllocator <IPhone> >();
block.Export <ZipExtractor>().As <IZipExtractor>();
block.ExportFactory(Tokenizer.Create).As <Tokenizer <LangToken> >().Lifestyle.Singleton();
block.Export <ScriptParser>().As <IScriptParser>().Lifestyle.Singleton();
block.ExportFactory(() => installOptionsProvider).As <IWindowsOptionsProvider>().Lifestyle.Singleton();
block.Export <LumiaDiskLayoutPreparer>().As <IDiskLayoutPreparer>().Lifestyle.Singleton();
block.Export <PhoneInfoReader>().As <IPhoneInfoReader>().Lifestyle.Singleton();
block.Export <WoaDeployer>().As <IWoaDeployer>().Lifestyle.Singleton();
block.Export <Tooling>().As <ITooling>().Lifestyle.Singleton();
block.Export <BootCreator>().As <IBootCreator>().Lifestyle.Singleton();
block.Export <DiskApi>().As <IDiskApi>().Lifestyle.Singleton();
block.Export <PhonePathBuilder>().As <IPathBuilder>().Lifestyle.Singleton();
block.ExportInstance(taskTypes).As <IEnumerable <Type> >();
block.Export <ScriptRunner>().As <IScriptRunner>().Lifestyle.Singleton();
block.Export <InstanceBuilder>().As <IInstanceBuilder>().Lifestyle.Singleton();
block.ExportFactory((IPhone p) => new DeviceProvider {
Device = p
}).As <IDeviceProvider>();
block.Export <FileSystemOperations>().As <IFileSystemOperations>().Lifestyle.Singleton();
block.Export <BcdInvokerFactory>().As <IBcdInvokerFactory>().Lifestyle.Singleton();
block.Export <WindowsDeployer>().As <IWindowsDeployer>().Lifestyle.Singleton();
block.ExportFactory(() => new HttpClient {
Timeout = TimeSpan.FromMinutes(30)
}).Lifestyle.Singleton();
block.ExportFactory(() => new GitHubClient(new ProductHeaderValue("WOADeployer"))).As <IGitHubClient>();
block.Export <Downloader>().As <IDownloader>().Lifestyle.Singleton();
block.Export <ProviderBasedWindowsDeployer>().As <IProviderBasedWindowsDeployer>();
block.Export <PartitionCleaner>().As <IPartitionCleaner>();
block.ExportFactory(() => AzureDevOpsClient.Create(new Uri("https://dev.azure.com"))).As <IAzureDevOpsBuildClient>();
return(block);
}
public async Task <IRemote> GetRemoteAsync(string repoUrl, ILogger logger)
{
using (logger.BeginScope($"Getting remote for repo {repoUrl}."))
{
// Normalize the url with the AzDO client prior to attempting to
// get a token. When we do coherency updates we build a repo graph and
// may end up traversing links to classic azdo uris.
string normalizedUrl = AzureDevOpsClient.NormalizeUrl(repoUrl);
Uri normalizedRepoUri = new Uri(normalizedUrl);
IGitRepo gitClient;
long installationId = await Context.GetInstallationId(normalizedUrl);
switch (normalizedRepoUri.Host)
{
case "github.com":
if (installationId == default)
{
throw new GithubApplicationInstallationException($"No installation is avaliable for repository '{normalizedUrl}'");
}
gitClient = new GitHubClient(null, await GitHubTokenProvider.GetTokenForInstallationAsync(installationId),
logger, null, Cache.Cache);
break;
case "dev.azure.com":
gitClient = new AzureDevOpsClient(null, await AzureDevOpsTokenProvider.GetTokenForRepository(normalizedUrl),
logger, null);
break;
default:
throw new NotImplementedException($"Unknown repo url type {normalizedUrl}");
}
;
return(new Remote(gitClient, new MaestroBarClient(Context, KustoClientProvider), logger));
}
}
public async Task ProcessFinishedReleasesAsync(CancellationToken cancellationToken)
{
Logger.LogInformation($"Starting ProcessFinishedReleasesAsync.");
var runningPipelines =
await StateManager.GetOrAddAsync <IReliableDictionary <int, IList <ReleasePipelineStatusItem> > >(RunningPipelineDictionaryName);
HashSet <BuildChannel> buildChannelsToAdd = new HashSet <BuildChannel>(new BuildChannelComparer());
using (ITransaction tx = StateManager.CreateTransaction())
{
var runningPipelinesEnumerable = await runningPipelines.CreateEnumerableAsync(tx, EnumerationMode.Unordered);
using (var asyncEnumerator = runningPipelinesEnumerable.GetAsyncEnumerator())
{
while (await asyncEnumerator.MoveNextAsync(cancellationToken))
{
int buildId = asyncEnumerator.Current.Key;
IList <ReleasePipelineStatusItem> releaseStatuses = asyncEnumerator.Current.Value;
int channelId = releaseStatuses.First().ChannelId;
List <ReleasePipelineStatusItem> unfinishedReleases = new List <ReleasePipelineStatusItem>();
foreach (ReleasePipelineStatusItem releaseStatus in releaseStatuses)
{
try
{
int releaseId = releaseStatus.ReleaseId;
AzureDevOpsClient azdoClient = await GetAzureDevOpsClientForAccount(releaseStatus.PipelineOrganization);
AzureDevOpsRelease release =
await azdoClient.GetReleaseAsync(releaseStatus.PipelineOrganization, releaseStatus.PipelineProject, releaseStatus.ReleaseId);
if (HasInProgressEnvironments(release))
{
unfinishedReleases.Add(releaseStatus);
Logger.LogInformation($"Release {releaseId} from build {buildId} and channel {channelId} is still in progress.");
}
else
{
Logger.LogInformation($"Release {releaseId}, channel {channelId} finished executing");
if (release.Environments.Any(r => r.Status != AzureDevOpsReleaseStatus.Succeeded))
{
await CreateGitHubIssueAsync(buildId, releaseId, release.Name);
await StateManager.RemoveAsync(release.Name);
}
}
}
catch (TaskCanceledException tcex) when(tcex.CancellationToken == cancellationToken)
{
// ignore
}
catch (Exception ex)
{
// Something failed while fetching the release information so the potential created issue wouldn't have relevant information to
// be notified so we just log the exception to AppInsights with not filed issue.
Logger.LogError(ex, $"Processing release {releaseStatus.ReleaseId} failed. Check the exception for details.");
}
}
if (unfinishedReleases.Count > 0)
{
await runningPipelines.TryUpdateAsync(tx, buildId, unfinishedReleases, releaseStatuses);
}
else
{
Logger.LogInformation($"All releases for build {buildId} for channel {channelId} finished. Creating BuildChannel.");
buildChannelsToAdd.Add(new BuildChannel
{
BuildId = buildId,
ChannelId = channelId
});
await runningPipelines.TryRemoveAsync(tx, buildId);
}
}
}
if (buildChannelsToAdd.Count > 0)
{
List <BuildChannel> addedBuildChannels = await AddFinishedBuildChannelsIfNotPresent(buildChannelsToAdd);
await TriggerDependencyUpdates(addedBuildChannels);
}
await tx.CommitAsync();
}
}
/// <summary>
/// Start release pipeline associated with a channel.
/// </summary>
/// <param name="buildId">Maestro build id.</param>
/// <param name="channelId">Maestro channel id.</param>
/// <returns></returns>
public async Task RunAssociatedReleasePipelinesAsync(int buildId, int channelId, CancellationToken cancellationToken)
{
Logger.LogInformation($"Starting release pipeline for {buildId} in {channelId}");
Build build = await Context.Builds
.Where(b => b.Id == buildId).FirstOrDefaultAsync();
if (build == null)
{
Logger.LogError($"Could not find the specified BAR Build {buildId} to run a release pipeline.");
return;
}
// If something uses the old API version we won't have this information available.
// This will also be the case if something adds an existing build (created using
// the old API version) to a channel
if (build.AzureDevOpsBuildId == null)
{
Logger.LogInformation($"barBuildInfo.AzureDevOpsBuildId is null for BAR Build.Id {build.Id}.");
return;
}
Channel channel = await Context.Channels
.Where(ch => ch.Id == channelId)
.Include(ch => ch.ChannelReleasePipelines)
.ThenInclude(crp => crp.ReleasePipeline)
.FirstOrDefaultAsync();
if (channel == null)
{
Logger.LogInformation($"Could not find the specified channel {channelId} to run a release pipeline on.");
return;
}
if (channel.ChannelReleasePipelines?.Any() != true)
{
Logger.LogInformation($"Channel {channel.Id}, which build with BAR ID {build.Id} is attached to, doesn't have an associated publishing pipeline.");
return;
}
AzureDevOpsClient azdoClient = await GetAzureDevOpsClientForAccount(build.AzureDevOpsAccount);
var azdoBuild = await azdoClient.GetBuildAsync(
build.AzureDevOpsAccount,
build.AzureDevOpsProject,
build.AzureDevOpsBuildId.Value);
var runningPipelines =
await StateManager.GetOrAddAsync <IReliableDictionary <int, IList <ReleasePipelineStatusItem> > >(RunningPipelineDictionaryName);
List <ReleasePipelineStatusItem> releaseList = new List <ReleasePipelineStatusItem>();
Logger.LogInformation($"Found {channel.ChannelReleasePipelines.Count} pipeline(s) for channel {channelId}");
foreach (ChannelReleasePipeline pipeline in channel.ChannelReleasePipelines)
{
try
{
string organization = pipeline.ReleasePipeline.Organization;
string project = pipeline.ReleasePipeline.Project;
int pipelineId = pipeline.ReleasePipeline.PipelineIdentifier;
Logger.LogInformation($"Going to create a release using pipeline {organization}/{project}/{pipelineId}");
AzureDevOpsReleaseDefinition pipeDef = await azdoClient.GetReleaseDefinitionAsync(organization, project, pipelineId);
pipeDef = await azdoClient.AdjustReleasePipelineArtifactSourceAsync(organization, project, pipeDef, azdoBuild);
int releaseId = await azdoClient.StartNewReleaseAsync(organization, project, pipeDef, build.Id);
var item = new ReleasePipelineStatusItem(releaseId, channelId, organization, project);
releaseList.Add(item);
Logger.LogInformation($"Created release {releaseId} using pipeline {organization}/{project}/{pipelineId}");
}
catch (Exception e)
{
Logger.LogError($"Some problem happened while starting publishing pipeline " +
$"{pipeline.ReleasePipeline.PipelineIdentifier} for build " +
$"{build.AzureDevOpsBuildId}: {e.Message}", e);
throw;
}
}
if (releaseList.Count > 0)
{
using (ITransaction tx = StateManager.CreateTransaction())
{
var runningPipelinesForBuild = await runningPipelines.TryGetValueAsync(tx, buildId);
if (runningPipelinesForBuild.HasValue)
{
// Some channel already triggered release pipelines for this build. Need to update with the releases for the new channel.
releaseList.AddRange(runningPipelinesForBuild.Value);
await runningPipelines.TryUpdateAsync(tx, buildId, releaseList, runningPipelinesForBuild.Value);
}
else
{
await runningPipelines.AddAsync(tx, buildId, releaseList);
}
await tx.CommitAsync();
}
}
}
public void NormalizeRepoUriTests(string inputUri, string expectedUri)
{
string normalizedUri = AzureDevOpsClient.NormalizeUrl(inputUri);
normalizedUri.Should().Be(expectedUri);
}
public void ParseInvalidPullRequestUriTests(string inputUri)
{
(((Func <object>)(() => AzureDevOpsClient.ParsePullRequestUri(inputUri)))).Should().ThrowExactly <ArgumentException>();
}
/// <summary>
/// Start release pipeline associated with a channel.
/// </summary>
/// <param name="buildId">Maestro build id.</param>
/// <param name="channelId">Maestro channel id.</param>
/// <returns></returns>
public async Task RunAssociatedReleasePipelinesAsync(int buildId, int channelId, CancellationToken cancellationToken)
{
Logger.LogInformation($"Starting release pipeline for {buildId} in {channelId}");
Build build = await Context.Builds
.Include(b => b.BuildChannels)
.Where(b => b.Id == buildId).FirstOrDefaultAsync();
if (build == null)
{
Logger.LogError($"Could not find the specified BAR Build {buildId} to run a release pipeline.");
return;
}
if (build.BuildChannels.Any(c => c.ChannelId == channelId))
{
Logger.LogInformation($"BAR build {buildId} is already in channel {channelId}. Skipping running releases for it.");
return;
}
// Check if we're already processing releases for this build in this channel
var runningPipelines =
await StateManager.GetOrAddAsync <IReliableDictionary <int, IList <ReleasePipelineStatusItem> > >(RunningPipelineDictionaryName);
using (ITransaction tx = StateManager.CreateTransaction())
{
var runningPipelinesForBuild = await runningPipelines.TryGetValueAsync(tx, buildId);
if (runningPipelinesForBuild.HasValue)
{
if (runningPipelinesForBuild.Value.Any(i => i.ChannelId == channelId))
{
Logger.LogInformation($"Releases already in progress for build {buildId} and channel {channelId}. Skipping running new releases.");
return;
}
}
}
// If something uses the old API version we won't have this information available.
// This will also be the case if something adds an existing build (created using
// the old API version) to a channel
if (build.AzureDevOpsBuildId == null)
{
Logger.LogInformation($"barBuildInfo.AzureDevOpsBuildId is null for BAR Build.Id {build.Id}.");
return;
}
Channel channel = await Context.Channels
.Where(ch => ch.Id == channelId)
.Include(ch => ch.ChannelReleasePipelines)
.ThenInclude(crp => crp.ReleasePipeline)
.FirstOrDefaultAsync();
if (channel == null)
{
Logger.LogInformation($"Could not find the specified channel {channelId} to run a release pipeline on.");
return;
}
if (channel.ChannelReleasePipelines?.Any() != true)
{
Logger.LogInformation($"Channel {channel.Id}, which build with BAR ID {build.Id} is attached to, doesn't have an associated publishing pipeline.");
return;
}
AzureDevOpsClient azdoClient = await GetAzureDevOpsClientForAccount(build.AzureDevOpsAccount);
var azdoBuild = await azdoClient.GetBuildAsync(
build.AzureDevOpsAccount,
build.AzureDevOpsProject,
build.AzureDevOpsBuildId.Value);
List <ReleasePipelineStatusItem> releaseList = new List <ReleasePipelineStatusItem>();
Logger.LogInformation($"Found {channel.ChannelReleasePipelines.Count} pipeline(s) for channel {channelId}");
if (channel.ChannelReleasePipelines.Select(pipeline =>
pipeline.ReleasePipeline.Organization).Distinct(StringComparer.OrdinalIgnoreCase).Count() > 1)
{
Logger.LogError($"Multiple pipelines in different organizations are not supported (channel {channel.Id}).");
return;
}
foreach (ChannelReleasePipeline pipeline in channel.ChannelReleasePipelines)
{
try
{
string organization = pipeline.ReleasePipeline.Organization;
string project = pipeline.ReleasePipeline.Project;
int pipelineId = pipeline.ReleasePipeline.PipelineIdentifier;
// If the release definition is in a separate organization or project than the
// build, running the release won't work. This is not that interesting anymore as the repos that have
// this issue are on stages. So we can just skip them.
if (!organization.Equals(build.AzureDevOpsAccount, StringComparison.OrdinalIgnoreCase) ||
!project.Equals(build.AzureDevOpsProject, StringComparison.OrdinalIgnoreCase))
{
Logger.LogWarning($"Skipping release of build {build.Id} because it is not in the same organzation or project as the release definition.");
await AddFinishedBuildChannelsIfNotPresent(new HashSet <BuildChannel> {
new BuildChannel
{
BuildId = buildId,
ChannelId = channelId,
DateTimeAdded = DateTimeOffset.UtcNow
}
});
break;
}
Logger.LogInformation($"Going to create a release using pipeline {organization}/{project}/{pipelineId}");
AzureDevOpsReleaseDefinition pipeDef = await azdoClient.GetReleaseDefinitionAsync(organization, project, pipelineId);
pipeDef = await azdoClient.AdjustReleasePipelineArtifactSourceAsync(organization, project, pipeDef, azdoBuild);
int releaseId = await azdoClient.StartNewReleaseAsync(organization, project, pipeDef, build.Id);
var item = new ReleasePipelineStatusItem(releaseId, channelId, organization, project);
releaseList.Add(item);
Logger.LogInformation($"Created release {releaseId} using pipeline {organization}/{project}/{pipelineId}");
}
catch (Exception e)
{
Logger.LogError($"Some problem happened while starting publishing pipeline " +
$"{pipeline.ReleasePipeline.PipelineIdentifier} for build " +
$"{build.AzureDevOpsBuildId}: {e.Message}", e);
throw;
}
}
if (releaseList.Count > 0)
{
using (ITransaction tx = StateManager.CreateTransaction())
{
var runningPipelinesForBuild = await runningPipelines.TryGetValueAsync(tx, buildId);
if (runningPipelinesForBuild.HasValue)
{
// Some channel already triggered release pipelines for this build. Need to update with the releases for the new channel.
releaseList.AddRange(runningPipelinesForBuild.Value);
await runningPipelines.TryUpdateAsync(tx, buildId, releaseList, runningPipelinesForBuild.Value);
}
else
{
await runningPipelines.AddAsync(tx, buildId, releaseList);
}
await tx.CommitAsync();
}
}
}
public async Task <TimeSpan> RunAsync(CancellationToken cancellationToken)
{
IReliableConcurrentQueue <ReleasePipelineRunnerItem> queue =
await StateManager.GetOrAddAsync <IReliableConcurrentQueue <ReleasePipelineRunnerItem> >("queue");
try
{
using (ITransaction tx = StateManager.CreateTransaction())
{
ConditionalValue <ReleasePipelineRunnerItem> maybeItem = await queue.TryDequeueAsync(
tx,
cancellationToken);
if (maybeItem.HasValue)
{
ReleasePipelineRunnerItem item = maybeItem.Value;
Build build = await Context.Builds
.Where(b => b.Id == item.BuildId).FirstOrDefaultAsync();
if (build == null)
{
Logger.LogError($"Could not find the specified BAR Build {item.BuildId} to run a release pipeline.");
}
else if (build.AzureDevOpsBuildId == null)
{
// If something uses the old API version we won't have this information available.
// This will also be the case if something adds an existing build (created using
// the old API version) to a channel
Logger.LogInformation($"barBuildInfo.AzureDevOpsBuildId is null for BAR Build.Id {build.Id}.");
}
else
{
AzureDevOpsClient azdoClient = await GetAzureDevOpsClientForAccount(build.AzureDevOpsAccount);
var azdoBuild = await azdoClient.GetBuildAsync(
build.AzureDevOpsAccount,
build.AzureDevOpsProject,
build.AzureDevOpsBuildId.Value);
if (azdoBuild.Status.Equals("completed", StringComparison.OrdinalIgnoreCase))
{
await HandleCompletedBuild(item, azdoBuild, cancellationToken);
}
else
{
Logger.LogInformation($"AzDO build {azdoBuild.BuildNumber}/{azdoBuild.Definition.Name} with BAR BuildId {build.Id} is still in progress.");
// Build didn't finish yet. Let's wait some time and try again.
EnqueueBuildStatusCheck(item, 0);
}
}
}
await tx.CommitAsync();
}
}
catch (TaskCanceledException tcex) when(tcex.CancellationToken == cancellationToken)
{
return(TimeSpan.MaxValue);
}
catch (Exception ex)
{
Logger.LogError(ex, "Processing queue messages");
}
return(TimeSpan.FromMinutes(1));
}
