private async Task <List <CiCdBuildDto> > GetBuildsByRepository(RepositoryDto repository)
{
List <CiCdBuildDto> builds = new List <CiCdBuildDto>();
IPaginatedList <CiCdDto> cicds = await ciCdService.GetAsync(pageIndex : 1, pageSize : 1000);
foreach (var cicd in cicds)
{
encryption.Decrypt(cicd);
}
foreach (ICiCdProvider provider in providers)
{
foreach (CiCdDto cicd in cicds.Where(cicd => provider.Supports(cicd.Kind)))
{
try
{
List <CiCdBuildDto> results = await provider.GetBuildsAsync(repository, cicd);
builds.AddRange(results);
}
catch (Exception e)
{
logger.LogError(e, $"{nameof(GetBuildsByRepositoryIdAsync)}::[{repository.Url}]::[{cicd.Endpoint}]::ERROR");
}
}
}
logger.LogInformation($"{nameof(GetBuildsByRepositoryIdAsync)}::[{repository.WebUrl}]::CICD::SEARCH::RESULTS::[{builds.Count}]");
cache.Set(repository.RepositoryId, builds, absoluteExpiration: DateTimeOffset.Now.AddHours(2));
return(builds);
}
public async Task RefreshRepositoryByIdAsync(Guid repositoryId)
{
VcsRepository repository = await context.VcsRepositories.FindAsync(repositoryId);
Vcs versionControl = await context.VcsSources.FindAsync(repository.VcsId);
logger.LogInformation($"Refreshing repository: {repository.Url}");
await context.Entry(repository).Collection(r => r.Assets).LoadAsync();
context.Assets.RemoveRange(repository.Assets);
await context.SaveChangesAsync();
var versionControlDto = new VersionControlDto {
ApiKey = versionControl.ApiKey, Endpoint = versionControl.Endpoint, VcsId = versionControl.Id, Kind = versionControl.Kind, IsEnabled = versionControl.IsEnabled
};
var repositoryDto = new RepositoryDto {
VcsId = repository.VcsId, Url = repository.Url, WebUrl = repository.WebUrl
};
encryptionService.Decrypt(versionControlDto);
IEnumerable <Asset> items = await versionControlProvider.GetAssetsAsync(versionControlDto, repositoryDto);
foreach (Asset asset in items)
{
context.Assets.Add(asset);
await context.SaveChangesAsync();
await RefreshAssetByIdAsync(asset.Id);
}
}
public Task <List <CiCdBuildDto> > GetBuildsAsync(RepositoryDto repository, CiCdDto cicd)
{
var picker = new UniqueRandomPicker <string>(With.Between(1).And(FakeBuilds.Length).Elements, new UniqueRandomGenerator());
var items = picker.From(FakeBuilds);
return(Task.FromResult(items.Select(x => new CiCdBuildDto {
Name = x, WebUrl = $"http://ci.rbxd.ds:8090/path/to/build/{x}"
}).ToList()));
}
protected override async Task <List <CiCdBuildDto> > TryGetBuildsAsync(RepositoryDto repository, CiCdDto cicd)
{
var builds = new List <CiCdBuildDto>();
try
{
var client = GetHttpClient(cicd);
var authMode = cicd.IsGuestEnabled ? "guestAuth" : "httpAuth";
var query = cicd.Endpoint.Trim('/') + $"/{authMode}/app/rest/vcs-roots";
logger.LogTrace($"Checking {query} for builds related to {repository.WebUrl}");
var vcsRootsResponse = await client.GetAsync(query, HttpCompletionOption.ResponseContentRead);
var vcsRootsDocument = XDocument.Parse(await vcsRootsResponse.Content.ReadAsStringAsync());
foreach (var vcsRootElement in vcsRootsDocument.Root.Elements("vcs-root"))
{
var uri = vcsRootElement.Attribute("href")?.Value;
try
{
var vcsRootResponse = await client.GetAsync(cicd.Endpoint.Trim('/') + uri);
var vscRootDocument = XDocument.Parse(await vcsRootResponse.Content.ReadAsStringAsync());
var projectElement = vscRootDocument.Root.Element("project");
var vcsElement = vscRootDocument.Root.Element("properties").Elements("property").FirstOrDefault(p => p.Attribute("name").Value == "url");
var name = projectElement.Attribute("name").Value;
var webUri = projectElement.Attribute("webUrl").Value;
var vcsUrl = vcsElement?.Attribute("value").Value;
if (matcher.IsMatch(vcsUrl, repository.Url))
{
builds.Add(new CiCdBuildDto {
Name = name, WebUrl = webUri, CiCdId = cicd.CiCdId, Kind = cicd.Kind
});
}
}
catch (Exception e)
{
logger.LogTrace(e, $"Error finding build at {uri}");
}
}
}
catch (Exception e)
{
logger.LogTrace(e, $"Error finding builds for {repository.WebUrl} at {cicd.Endpoint}");
}
return(builds);
}
public async Task <List <CiCdBuildDto> > GetBuildsByRepositoryIdAsync(Guid repositoryId)
{
RepositoryDto repository = await repositoryService.GetByIdAsync(repositoryId);
using (var scope = logger.BeginScope($"{nameof(GetBuildsByRepositoryIdAsync)}::[{repository.Url}]::CICD::SEARCH"))
{
if (!cache.TryGetValue(repositoryId, out List <CiCdBuildDto> builds))
{
builds = await GetBuildsByRepository(repository);
}
return(await Task.FromResult(builds));
}
}
public async Task <List <CiCdBuildDto> > GetBuildsAsync(RepositoryDto repository, CiCdDto cicd)
{
using (var scope = logger.BeginScope($"[{repository.Url}]::[{cicd.Endpoint}]"))
{
try
{
return(await TryGetBuildsAsync(repository, cicd));
}
catch (Exception e)
{
logger.LogError(e, $"[{repository.Url}]::[{cicd.Endpoint}]::ERROR");
return(null);
}
}
}
protected override async Task <List <CiCdBuildDto> > TryGetBuildsAsync(RepositoryDto repository, CiCdDto cicd)
{
var results = new List <CiCdBuildDto>();
try
{
HttpClient httpClient = GetHttpClient(cicd);
logger.LogTrace($"Checking {cicd.Endpoint} for builds related to {repository.WebUrl}");
var jobs = await GetJobs(httpClient, new Uri(cicd.Endpoint));
var builds = await GetBuilds(httpClient, jobs);
foreach (var build in builds)
{
try
{
var buildInfo = XDocument.Parse(await HttpClient.GetStringAsync(new Uri(new Uri(build, UriKind.Absolute), "api/xml")));
var vcsUrl = buildInfo.XPathSelectElement("//remoteUrl").Value;
var name = buildInfo.XPathSelectElement("//fullDisplayName").Value;
var webUrl = buildInfo.XPathSelectElement("//url").Value;
if (matcher.IsMatch(vcsUrl, repository.Url))
{
results.Add(new CiCdBuildDto {
Name = name, WebUrl = webUrl, CiCdId = cicd.CiCdId, Kind = cicd.Kind
});
}
}
catch (Exception e)
{
logger.LogTrace(e, $"Error finding build at {build}");
}
}
}
catch (Exception e)
{
logger.LogTrace(e, $"Error finding builds for {repository.WebUrl} at {cicd.Endpoint}");
}
return(results);
}
public async Task <IEnumerable <Asset> > GetAssetsAsync(VersionControlDto versionControl, RepositoryDto repository)
{
using (var scope = logger.BeginScope($"{nameof(GetAssetsAsync)}::[{repository.Url}]"))
{
List <Asset> results = new List <Asset>();
foreach (IVersionControlProvider provider in providers.Where(provider => provider.Kind == versionControl.Kind))
{
try
{
IEnumerable <Asset> assets = await provider.GetAssetsAsync(versionControl, repository);
results.AddRange(assets);
}
catch (Exception e)
{
logger.LogError(e, $"{nameof(GetAssetsAsync)}::[{repository.Url}]::ERROR");
}
}
logger.LogInformation($"{nameof(GetAssetsAsync)}::[{repository.Url}]::RESULTS::{results.Count}");
return(results);
}
}
public async Task <IEnumerable <Asset> > GetAssetsAsync(VersionControlDto versionControl, RepositoryDto repository)
{
return(await Task.FromResult(Enumerable.Empty <Asset>()));
}
public async Task <IEnumerable <Asset> > GetAssetsAsync(VersionControlDto versionControl, RepositoryDto repository)
{
List <Asset> results = new List <Asset>();
StashClient client = new StashClient(versionControl.Endpoint, versionControl.ApiKey, usePersonalAccessTokenForAuthentication: true);
ResponseWrapper <Project> projects = await client.Projects.Get();
foreach (Project project in projects.Values ?? Enumerable.Empty <Project>())
{
ResponseWrapper <Repository> repositories = await client.Repositories.Get(project.Key, Options);
foreach (Repository bitBucketRepository in repositories.Values ?? Enumerable.Empty <Repository>())
{
bool isRepositoryFound = bitBucketRepository.Links.Clone.Select(c => c.Href).Concat(bitBucketRepository.Links.Self.Select(s => s.Href)).Any(link => matcher.IsMatch(link.ToString(), repository.Url));
if (isRepositoryFound)
{
ResponseWrapper <string> filePaths = await client.Repositories.GetFiles(project.Key, bitBucketRepository.Slug, Options);
foreach (string path in filePaths.Values ?? Enumerable.Empty <string>())
{
if (path.IsSupported())
{
File file = await client.Repositories.GetFileContents(project.Key, bitBucketRepository.Slug, path, new FileContentsOptions { Content = true, Limit = 10000 });
logger.LogInformation($"Adding '{path}' for repository {repository.RepositoryId}");
results.Add(new Asset {
Id = Guid.NewGuid(), RepositoryId = repository.RepositoryId, Kind = path.GetEcosystemKind(), Path = path, Raw = string.Join(Environment.NewLine, file.FileContents)
});
}
}
}
}
}
return(results);
}
protected abstract Task <List <CiCdBuildDto> > TryGetBuildsAsync(RepositoryDto repository, CiCdDto cicd);