public async Task <List <string> > GetOwnerIdsForUserAsync(IIdentity identity)
{
if (identity == null)
{
throw new ArgumentNullException();
}
var ownerIdList = new List <string> {
identity.Name
};
try
{
var ghClient = _gitHubClientFactory.CreateClient(identity as ClaimsIdentity);
var orgs = await ghClient.Organization.GetAllForCurrent();
if (orgs == null || !orgs.Any())
{
Log.Warning("GetOwnerIdsForuserAsync: No organizations returned for user '{0}'", identity.Name);
return(ownerIdList);
}
Log.Debug("GetOwnerIdsForuserAsync: {0} organizations found for user '{1}'", orgs.Count(), identity.Name);
ownerIdList.AddRange(orgs.Select(o => o.Login));
return(ownerIdList);
}
catch (Exception ex)
{
Log.Error(ex, "GetOwnerIdsForuserAsync: Error retrieving organization list for user {0}.", identity.Name);
Log.Error(ex.ToString());
return(null);
}
}
/// <inheritdoc />
public async Task <string> ValidateResponseCode(string code, CancellationToken cancellationToken)
{
if (code == null)
{
throw new ArgumentNullException(nameof(code));
}
var client = gitHubClientFactory.CreateClient();
try
{
logger.LogTrace("Validating response code...");
var response = await client
.Oauth
.CreateAccessToken(
new OauthTokenRequest(
oAuthConfiguration.ClientId,
oAuthConfiguration.ClientSecret,
code)
{
RedirectUri = oAuthConfiguration.RedirectUrl,
})
.ConfigureAwait(false);
var token = response.AccessToken;
if (token == null)
{
return(null);
}
var authenticatedClient = gitHubClientFactory.CreateClient(token);
logger.LogTrace("Getting user details...");
var userDetails = await authenticatedClient
.User
.Current()
.ConfigureAwait(false);
return(userDetails.Id.ToString(CultureInfo.InvariantCulture));
}
catch (RateLimitExceededException)
{
throw;
}
catch (ApiException ex)
{
logger.LogWarning(ex, "API error while completing OAuth handshake!");
return(null);
}
}
public async Task <ChangeLogTaskResult> RunAsync(ApplicationChangeLog changeLog)
{
if (m_ProjectInfo == null)
{
return(ChangeLogTaskResult.Skipped);
}
m_Logger.LogInformation("Adding GitHub links to changelog");
var githubClient = m_GitHubClientFactory.CreateClient(m_ProjectInfo.Host);
var rateLimit = await githubClient.Miscellaneous.GetRateLimits();
m_Logger.LogDebug($"GitHub rate limit: {rateLimit.Rate.Remaining} requests of {rateLimit.Rate.Limit} remaining");
foreach (var versionChangeLog in changeLog.ChangeLogs)
{
foreach (var entry in versionChangeLog.AllEntries)
{
await ProcessEntryAsync(githubClient, entry);
}
}
return(ChangeLogTaskResult.Success);
}
public async Task <IActionResult> Read(CancellationToken cancellationToken)
{
try
{
Version greatestVersion = null;
Uri repoUrl = null;
try
{
var gitHubClient = gitHubClientFactory.CreateClient();
var repositoryTask = gitHubClient
.Repository
.Get(updatesConfiguration.GitHubRepositoryId)
.WithToken(cancellationToken);
var releases = (await gitHubClient
.Repository
.Release
.GetAll(updatesConfiguration.GitHubRepositoryId)
.WithToken(cancellationToken)
.ConfigureAwait(false))
.Where(x => x.TagName.StartsWith(
updatesConfiguration.GitTagPrefix,
StringComparison.InvariantCulture));
foreach (var release in releases)
{
if (Version.TryParse(release.TagName.Replace(updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), out var version) &&
version.Major == assemblyInformationProvider.Version.Major &&
(greatestVersion == null || version > greatestVersion))
{
greatestVersion = version;
}
}
repoUrl = new Uri((await repositoryTask.ConfigureAwait(false)).HtmlUrl);
}
catch (NotFoundException e)
{
Logger.LogWarning(e, "Not found exception while retrieving upstream repository info!");
}
return(Json(new AdministrationResponse
{
LatestVersion = greatestVersion,
TrackedRepositoryUrl = repoUrl,
}));
}
catch (RateLimitExceededException e)
{
return(RateLimit(e));
}
catch (ApiException e)
{
Logger.LogWarning(e, OctokitException);
return(StatusCode(HttpStatusCode.FailedDependency, new ErrorMessageResponse(ErrorCode.RemoteApiError)
{
AdditionalData = e.Message,
}));
}
}
public async Task <ReleaseInfo> MakeRelease(IGraph dataGraph, string releaseTag, string owner, string repositoryId, string datasetId, string repositoryDirectory, string authenticationToken)
{
var releaseInfo = new ReleaseInfo(releaseTag);
var ntriplesDumpFileName = Path.Combine(repositoryDirectory, releaseTag + ".nt.gz");
ProgressLog.Info("Generating gzipped NTriples data dump");
var writer = new GZippedNTriplesWriter();
writer.Save(dataGraph, ntriplesDumpFileName);
// Make a release
try
{
ProgressLog.Info("Generating a new release of dataset {0}", datasetId);
if (authenticationToken == null)
{
throw new WorkerException("No valid GitHub access token found for your account.");
}
var client = _gitHubClientFactory.CreateClient(authenticationToken);
client.SetRequestTimeout(TimeSpan.FromSeconds(300));
var releaseClient = client.Repository.Release;
var newRelease = new NewRelease(releaseTag)
{
TargetCommitish = "gh-pages"
};
var release = await releaseClient.Create(owner, repositoryId, newRelease);
// Attach data dump file(s) to release
try
{
ProgressLog.Info("Uploading data dump files to GitHub release");
using (var zipFileStream = File.OpenRead(ntriplesDumpFileName))
{
var upload = new ReleaseAssetUpload(Path.GetFileName(ntriplesDumpFileName), "application/gzip",
zipFileStream, null);
var releaseAsset = await releaseClient.UploadAsset(release, upload);
releaseInfo.DownloadLinks.Add(releaseAsset.BrowserDownloadUrl);
}
}
catch (Exception ex)
{
Log.Error(ex, "Failed to attach dump files to GitHub release");
throw new WorkerException(ex, "Failed to attach dump files to GitHub release");
}
}
catch (WorkerException)
{
throw;
}
catch (Exception ex)
{
Log.Error(ex, "Failed to create a new GitHub release");
throw new WorkerException(ex, "Failed to create a new GitHub release");
}
return(releaseInfo);
}
public async Task <ChangeLogTaskResult> RunAsync(ApplicationChangeLog changeLog)
{
var projectInfo = GetProjectInfo();
if (projectInfo != null)
{
m_Logger.LogDebug($"Enabling GitHub integration with settings: " +
$"{nameof(projectInfo.Host)} = '{projectInfo.Host}', " +
$"{nameof(projectInfo.Owner)} = '{projectInfo.Owner}', " +
$"{nameof(projectInfo.Repository)} = '{projectInfo.Repository}'");
}
else
{
m_Logger.LogWarning("Failed to determine GitHub project name. Disabling GitHub integration");
return(ChangeLogTaskResult.Skipped);
}
m_Logger.LogInformation("Adding GitHub links to change log");
var githubClient = m_GitHubClientFactory.CreateClient(projectInfo.Host);
var rateLimit = await githubClient.Miscellaneous.GetRateLimits();
m_Logger.LogDebug($"GitHub rate limit: {rateLimit.Rate.Remaining} requests of {rateLimit.Rate.Limit} remaining");
try
{
foreach (var versionChangeLog in changeLog.ChangeLogs)
{
foreach (var entry in versionChangeLog.AllEntries)
{
await ProcessEntryAsync(projectInfo, githubClient, entry);
}
}
return(ChangeLogTaskResult.Success);
}
catch (RateLimitExceededException rateLimitExceededException)
{
var messageBuilder = new StringBuilder();
messageBuilder.Append($"GitHub API rate limit exceeded (limit { rateLimitExceededException.Limit}). ");
if (githubClient.Connection.Credentials.AuthenticationType == AuthenticationType.Anonymous)
{
messageBuilder.Append("Consider using an Access Token for GitHub. Authenticated requests are given a higher rate limit.");
}
m_Logger.LogError(rateLimitExceededException, messageBuilder.ToString());
return(ChangeLogTaskResult.Error);
}
catch (ApiException ex)
{
m_Logger.LogError(ex, ex.Message);
return(ChangeLogTaskResult.Error);
}
}
public UpdateService(
IDownloader fileDownloader,
IFileManager fileManager,
IProcessManager processManager,
ILogger logger,
IGitHubClientFactory clientFactory)
{
client = clientFactory.CreateClient();
this.fileDownloader = fileDownloader;
this.fileManager = fileManager;
this.processManager = processManager;
this.logger = logger;
}
public UpdateService(
IDownloader fileDownloader,
IFileManager fileManager,
IProcessManager processManager,
ILogger logger,
IGitHubClientFactory clientFactory)
{
client = clientFactory.CreateClient();
this.fileDownloader = fileDownloader;
this.fileManager = fileManager;
this.processManager = processManager;
this.logger = logger;
}
public static async Task <IActionResult> RunGitHub(
[HttpTrigger(AuthorizationLevel.Function, "get")] HttpRequest req,
ILogger logger,
[Inject] IGitHubClientFactory factory)
{
logger.LogInformation("RunGitHub fired - HTTP");
var client = factory.CreateClient();
string userid = req.Query.ContainsKey("userid") ? req.Query["userid"].ToString() : "stefh";
string result = await client.GetAsync(userid);
logger.LogInformation("result = " + result);
return(new OkResult());
}
public UpdateService(
IDownloader fileDownloader,
IFileManager fileManager,
IProcessManager processManager,
ILogger logger,
IGitHubClientFactory clientFactory,
IEnvironmentInformation environmentInformation)
{
client = clientFactory.CreateClient();
this.fileDownloader = fileDownloader;
this.fileManager = fileManager;
this.processManager = processManager;
this.logger = logger;
this.environmentInformation = environmentInformation;
}
public PullRequestService(
IOptions <WebConfiguration> config,
ISerializerFactory serializerFactory,
IGitHubClientFactory githubFactory)
{
if (serializerFactory is null)
{
throw new ArgumentNullException(nameof(serializerFactory));
}
if (githubFactory is null)
{
throw new ArgumentNullException(nameof(githubFactory));
}
_config = config?.Value ?? throw new ArgumentNullException(nameof(config));
_serializer = serializerFactory.BuildSerializer();
_github = githubFactory.CreateClient();
}
public UpdateService(
IDownloader fileDownloader,
IFileManager fileManager,
IProcessManager processManager,
ILogger logger,
IGitHubClientFactory clientFactory,
IEnvironmentInformation environmentInformation,
ISettingsManager settingsManager,
IMaintenanceWindow maintenanceWindow)
{
client = clientFactory.CreateClient();
this.fileDownloader = fileDownloader;
this.fileManager = fileManager;
this.processManager = processManager;
this.logger = logger;
this.environmentInformation = environmentInformation;
this.settingsManager = settingsManager;
this.maintenanceWindow = maintenanceWindow;
}
/// <inheritdoc />
public async Task <ServerUpdateResult> BeginUpdate(Version newVersion, CancellationToken cancellationToken)
{
logger.LogDebug("Looking for GitHub releases version {0}...", newVersion);
IEnumerable <Release> releases;
var gitHubClient = gitHubClientFactory.CreateClient();
releases = await gitHubClient
.Repository
.Release
.GetAll(updatesConfiguration.GitHubRepositoryId)
.WithToken(cancellationToken)
.ConfigureAwait(false);
releases = releases.Where(x => x.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture));
logger.LogTrace("Release query complete!");
foreach (var release in releases)
{
if (Version.TryParse(
release.TagName.Replace(
updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal),
out var version) &&
version == newVersion)
{
var asset = release.Assets.Where(x => x.Name.Equals(updatesConfiguration.UpdatePackageAssetName, StringComparison.Ordinal)).FirstOrDefault();
if (asset == default)
{
continue;
}
if (!serverControl.ApplyUpdate(version, new Uri(asset.BrowserDownloadUrl), ioManager))
{
return(ServerUpdateResult.UpdateInProgress);
}
return(ServerUpdateResult.Started);
}
}
return(ServerUpdateResult.ReleaseMissing);
}
public async Task Authenticate(string username, string password, LfsPermission requiredPermission, CancellationToken token)
{
var client = clientFactory.CreateClient(config.BaseAddress, username, password);
var repository = await client.Get(config.Organisation, config.Repository);
LfsPermission actualPermission = LfsPermission.None;
if (repository.Permissions.Pull)
{
actualPermission |= LfsPermission.Read;
}
if (repository.Permissions.Push)
{
actualPermission |= LfsPermission.Write;
}
if (!actualPermission.HasFlag(requiredPermission))
{
throw new InvalidOperationException($"User {username} doesn't have permission {requiredPermission} for repository {config.Organisation}/{config.Repository} (actual: {actualPermission})");
}
}
/// <inheritdoc />
public async Task StartDeployment(IRepository repository, CompileJob compileJob, CancellationToken cancellationToken)
{
if (repository == null)
{
throw new ArgumentNullException(nameof(repository));
}
if (compileJob == null)
{
throw new ArgumentNullException(nameof(compileJob));
}
if (!repository.IsGitHubRepository)
{
logger.LogTrace("Not managing deployment as this is not a GitHub repo");
return;
}
logger.LogTrace("Starting deployment...");
RepositorySettings repositorySettings = null;
await databaseContextFactory.UseContext(
async databaseContext =>
repositorySettings = await databaseContext
.RepositorySettings
.AsQueryable()
.Where(x => x.InstanceId == metadata.Id)
.FirstAsync(cancellationToken)
.ConfigureAwait(false))
.ConfigureAwait(false);
var gitHubClient = repositorySettings.AccessToken == null
? gitHubClientFactory.CreateClient()
: gitHubClientFactory.CreateClient(repositorySettings.AccessToken);
var repositoryTask = gitHubClient
.Repository
.Get(
repository.GitHubOwner,
repository.GitHubRepoName);
if (repositorySettings.CreateGitHubDeployments.Value)
{
logger.LogTrace("Creating deployment...");
var deployment = await gitHubClient
.Repository
.Deployment
.Create(
repository.GitHubOwner,
repository.GitHubRepoName,
new NewDeployment(compileJob.RevisionInformation.CommitSha)
{
AutoMerge = false,
Description = "TGS Game Deployment",
Environment = $"TGS: {metadata.Name}",
ProductionEnvironment = true,
RequiredContexts = new Collection <string>()
})
.WithToken(cancellationToken)
.ConfigureAwait(false);
compileJob.GitHubDeploymentId = deployment.Id;
logger.LogDebug("Created deployment ID {0}", deployment.Id);
await gitHubClient
.Repository
.Deployment
.Status
.Create(
repository.GitHubOwner,
repository.GitHubRepoName,
deployment.Id,
new NewDeploymentStatus(DeploymentState.InProgress)
{
Description = "The project is being deployed",
AutoInactive = false
})
.WithToken(cancellationToken)
.ConfigureAwait(false);
logger.LogTrace("In-progress deployment status created");
}
else
{
logger.LogTrace("Not creating deployment");
}
try
{
var gitHubRepo = await repositoryTask
.WithToken(cancellationToken)
.ConfigureAwait(false);
compileJob.GitHubRepoId = gitHubRepo.Id;
logger.LogTrace("Set GitHub ID as {0}", compileJob.GitHubRepoId);
}
catch (RateLimitExceededException ex) when(!repositorySettings.CreateGitHubDeployments.Value)
{
logger.LogWarning(ex, "Unable to set compile job repository ID!");
}
}
IGitHubClient GetGitHubClient() => String.IsNullOrEmpty(generalConfiguration.GitHubAccessToken) ? gitHubClientFactory.CreateClient() : gitHubClientFactory.CreateClient(generalConfiguration.GitHubAccessToken);
public void CreateGitHubClientWhenTokenAlreadyGot()
{
GitHubClientFactory.CreateClient(UserManager.CurrentUser.Token);
}
#pragma warning disable CA1502 // TODO: Decomplexify
#pragma warning disable CA1505
public async Task <IActionResult> Update([FromBody] Repository model, CancellationToken cancellationToken)
{
if (model == null)
{
throw new ArgumentNullException(nameof(model));
}
if (model.AccessUser == null ^ model.AccessToken == null)
{
return(BadRequest(new ErrorMessage {
Message = "Either both accessToken and accessUser must be present or neither!"
}));
}
if (model.CheckoutSha != null && model.Reference != null)
{
return(BadRequest(new ErrorMessage {
Message = "Only one of sha or reference may be specified!"
}));
}
if (model.CheckoutSha != null && model.UpdateFromOrigin == true)
{
return(BadRequest(new ErrorMessage {
Message = "Cannot update a reference when checking out a sha!"
}));
}
if (model.Origin != null)
{
return(BadRequest(new ErrorMessage {
Message = "origin cannot be modified without deleting the repository!"
}));
}
if (model.NewTestMerges?.Any(x => !x.Number.HasValue) == true)
{
return(BadRequest(new ErrorMessage {
Message = "All new test merges must provide a number!"
}));
}
if (model.NewTestMerges?.Any(x => model.NewTestMerges.Any(y => x != y && x.Number == y.Number)) == true)
{
return(BadRequest(new ErrorMessage {
Message = "Cannot test merge the same PR twice in one job!"
}));
}
if (model.CommitterName?.Length == 0)
{
return(BadRequest(new ErrorMessage {
Message = "Cannot set empty committer name!"
}));
}
if (model.CommitterEmail?.Length == 0)
{
return(BadRequest(new ErrorMessage {
Message = "Cannot set empty committer e=mail!"
}));
}
var newTestMerges = model.NewTestMerges != null && model.NewTestMerges.Count > 0;
var userRights = (RepositoryRights)AuthenticationContext.GetRight(RightsType.Repository);
if (newTestMerges && !userRights.HasFlag(RepositoryRights.MergePullRequest))
{
return(Forbid());
}
var currentModel = await DatabaseContext.RepositorySettings.Where(x => x.InstanceId == Instance.Id).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (currentModel == default)
{
return(StatusCode((int)HttpStatusCode.Gone));
}
bool CheckModified <T>(Expression <Func <Api.Models.Internal.RepositorySettings, T> > expression, RepositoryRights requiredRight)
{
var memberSelectorExpression = (MemberExpression)expression.Body;
var property = (PropertyInfo)memberSelectorExpression.Member;
var newVal = property.GetValue(model);
if (newVal == null)
{
return(false);
}
if (!userRights.HasFlag(requiredRight) && property.GetValue(currentModel) != newVal)
{
return(true);
}
property.SetValue(currentModel, newVal);
return(false);
}
if (CheckModified(x => x.AccessToken, RepositoryRights.ChangeCredentials) ||
CheckModified(x => x.AccessUser, RepositoryRights.ChangeCredentials) ||
CheckModified(x => x.AutoUpdatesKeepTestMerges, RepositoryRights.ChangeAutoUpdateSettings) ||
CheckModified(x => x.AutoUpdatesSynchronize, RepositoryRights.ChangeAutoUpdateSettings) ||
CheckModified(x => x.CommitterEmail, RepositoryRights.ChangeCommitter) ||
CheckModified(x => x.CommitterName, RepositoryRights.ChangeCommitter) ||
CheckModified(x => x.PushTestMergeCommits, RepositoryRights.ChangeTestMergeCommits) ||
CheckModified(x => x.ShowTestMergeCommitters, RepositoryRights.ChangeTestMergeCommits) ||
CheckModified(x => x.PostTestMergeComment, RepositoryRights.ChangeTestMergeCommits) ||
(model.UpdateFromOrigin == true && !userRights.HasFlag(RepositoryRights.UpdateBranch)))
{
return(Forbid());
}
if (currentModel.AccessToken?.Length == 0 && currentModel.AccessUser?.Length == 0)
{
// setting an empty string clears everything
currentModel.AccessUser = null;
currentModel.AccessToken = null;
}
var canRead = userRights.HasFlag(RepositoryRights.Read);
var api = canRead ? currentModel.ToApi() : new Repository();
var repoManager = instanceManager.GetInstance(Instance).RepositoryManager;
if (canRead)
{
if (repoManager.CloneInProgress)
{
return(Conflict(new ErrorMessage
{
Message = "A clone operation is in progress!"
}));
}
if (repoManager.InUse)
{
return(Conflict(new ErrorMessage
{
Message = "The repo is busy!"
}));
}
using (var repo = await repoManager.LoadRepository(cancellationToken).ConfigureAwait(false))
{
if (repo == null)
{
return(Conflict(new ErrorMessage
{
Message = "Repository could not be loaded!"
}));
}
await PopulateApi(api, repo, DatabaseContext, Instance, cancellationToken).ConfigureAwait(false);
}
}
// this is just db stuf so stow it away
await DatabaseContext.Save(cancellationToken).ConfigureAwait(false);
// format the job description
string description = null;
if (model.UpdateFromOrigin == true)
{
if (model.Reference != null)
{
description = String.Format(CultureInfo.InvariantCulture, "Fetch and hard reset repository to origin/{0}", model.Reference);
}
else if (model.CheckoutSha != null)
{
description = String.Format(CultureInfo.InvariantCulture, "Fetch and checkout {0} in repository", model.CheckoutSha);
}
else
{
description = "Pull current repository reference";
}
}
else if (model.Reference != null || model.CheckoutSha != null)
{
description = String.Format(CultureInfo.InvariantCulture, "Checkout repository {0} {1}", model.Reference != null ? "reference" : "SHA", model.Reference ?? model.CheckoutSha);
}
if (newTestMerges)
{
description = String.Format(CultureInfo.InvariantCulture, "{0}est merge pull request(s) {1}{2}",
description != null ? String.Format(CultureInfo.InvariantCulture, "{0} and t", description) : "T",
String.Join(", ", model.NewTestMerges.Select(x =>
String.Format(CultureInfo.InvariantCulture, "#{0}{1}", x.Number,
x.PullRequestRevision != null ? String.Format(CultureInfo.InvariantCulture, " at {0}", x.PullRequestRevision.Substring(0, 7)) : String.Empty))),
description != null ? String.Empty : " in repository");
}
if (description == null)
{
return(Json(api)); // no git changes
}
var job = new Models.Job
{
Description = description,
StartedBy = AuthenticationContext.User,
Instance = Instance,
CancelRightsType = RightsType.Repository,
CancelRight = (ulong)RepositoryRights.CancelPendingChanges,
};
await jobManager.RegisterOperation(job, async (paramJob, databaseContext, progressReporter, ct) =>
{
using (var repo = await repoManager.LoadRepository(ct).ConfigureAwait(false))
{
if (repo == null)
{
throw new JobException("Repository could not be loaded!");
}
var modelHasShaOrReference = model.CheckoutSha != null || model.Reference != null;
var startReference = repo.Reference;
var startSha = repo.Head;
string postUpdateSha = null;
if (newTestMerges && !repo.IsGitHubRepository)
{
throw new JobException("Cannot test merge on a non GitHub based repository!");
}
var committerName = currentModel.ShowTestMergeCommitters.Value ? AuthenticationContext.User.Name : currentModel.CommitterName;
var hardResettingToOriginReference = model.UpdateFromOrigin == true && model.Reference != null;
var numSteps = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0) + (!modelHasShaOrReference ? 2 : (hardResettingToOriginReference ? 3 : 1));
var doneSteps = 0;
Action <int> NextProgressReporter()
{
var tmpDoneSteps = doneSteps;
++doneSteps;
return(progress => progressReporter((progress + (100 * tmpDoneSteps)) / numSteps));
}
progressReporter(0);
// get a base line for where we are
Models.RevisionInformation lastRevisionInfo = null;
var attachedInstance = new Models.Instance
{
Id = Instance.Id
};
databaseContext.Instances.Attach(attachedInstance);
await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false);
// apply new rev info, tracking applied test merges
async Task UpdateRevInfo()
{
var last = lastRevisionInfo;
await LoadRevisionInformation(repo, databaseContext, attachedInstance, last.OriginCommitSha, x => lastRevisionInfo = x, ct).ConfigureAwait(false);
lastRevisionInfo.ActiveTestMerges.AddRange(last.ActiveTestMerges);
}
try
{
// fetch/pull
if (model.UpdateFromOrigin == true)
{
if (!repo.Tracking)
{
throw new JobException("Not on an updatable reference!");
}
await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct).ConfigureAwait(false);
doneSteps = 1;
if (!modelHasShaOrReference)
{
var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, NextProgressReporter(), ct).ConfigureAwait(false);
if (!fastForward.HasValue)
{
throw new JobException("Merge conflict occurred during origin update!");
}
await UpdateRevInfo().ConfigureAwait(false);
if (fastForward.Value)
{
lastRevisionInfo.OriginCommitSha = repo.Head;
await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false);
postUpdateSha = repo.Head;
}
else
{
NextProgressReporter()(100);
}
}
}
// checkout/hard reset
if (modelHasShaOrReference)
{
if ((model.CheckoutSha != null && repo.Head.ToUpperInvariant().StartsWith(model.CheckoutSha.ToUpperInvariant(), StringComparison.Ordinal)) ||
(model.Reference != null && repo.Reference.ToUpperInvariant() != model.Reference.ToUpperInvariant()))
{
var committish = model.CheckoutSha ?? model.Reference;
var isSha = await repo.IsSha(committish, cancellationToken).ConfigureAwait(false);
if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null))
{
throw new JobException("Attempted to checkout a SHA or reference that was actually the opposite!");
}
await repo.CheckoutObject(committish, NextProgressReporter(), ct).ConfigureAwait(false);
await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); // we've either seen origin before or what we're checking out is on origin
}
else
{
NextProgressReporter()(100);
}
if (hardResettingToOriginReference)
{
if (!repo.Tracking)
{
throw new JobException("Checked out reference does not track a remote object!");
}
await repo.ResetToOrigin(NextProgressReporter(), ct).ConfigureAwait(false);
await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false);
await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false);
// repo head is on origin so force this
// will update the db if necessary
lastRevisionInfo.OriginCommitSha = repo.Head;
}
}
// test merging
Dictionary <int, Octokit.PullRequest> prMap = null;
if (newTestMerges)
{
// bit of sanitization
foreach (var I in model.NewTestMerges.Where(x => String.IsNullOrWhiteSpace(x.PullRequestRevision)))
{
I.PullRequestRevision = null;
}
var gitHubClient = currentModel.AccessToken != null
? gitHubClientFactory.CreateClient(currentModel.AccessToken)
: (String.IsNullOrEmpty(generalConfiguration.GitHubAccessToken)
? gitHubClientFactory.CreateClient()
: gitHubClientFactory.CreateClient(generalConfiguration.GitHubAccessToken));
var repoOwner = repo.GitHubOwner;
var repoName = repo.GitHubRepoName;
// optimization: if we've already merged these exact same commits in this fashion before, just find the rev info for it and check it out
Models.RevisionInformation revInfoWereLookingFor = null;
bool needToApplyRemainingPrs = true;
if (lastRevisionInfo.OriginCommitSha == lastRevisionInfo.CommitSha)
{
// In order for this to work though we need the shas of all the commits
if (model.NewTestMerges.Any(x => x.PullRequestRevision == null))
{
prMap = new Dictionary <int, Octokit.PullRequest>();
}
bool cantSearch = false;
foreach (var I in model.NewTestMerges)
{
if (I.PullRequestRevision != null)
#pragma warning disable CA1308 // Normalize strings to uppercase
{
I.PullRequestRevision = I.PullRequestRevision?.ToLowerInvariant(); // ala libgit2
}
#pragma warning restore CA1308 // Normalize strings to uppercase
else
{
try
{
// retrieve the latest sha
var pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number.Value).ConfigureAwait(false);
prMap.Add(I.Number.Value, pr);
I.PullRequestRevision = pr.Head.Sha;
}
catch
{
cantSearch = true;
break;
}
}
}
if (!cantSearch)
{
var dbPull = await databaseContext.RevisionInformations
.Where(x => x.Instance.Id == Instance.Id &&
x.OriginCommitSha == lastRevisionInfo.OriginCommitSha &&
x.ActiveTestMerges.Count <= model.NewTestMerges.Count &&
x.ActiveTestMerges.Count > 0)
.Include(x => x.ActiveTestMerges)
.ThenInclude(x => x.TestMerge)
.ToListAsync(cancellationToken).ConfigureAwait(false);
// split here cause this bit has to be done locally
revInfoWereLookingFor = dbPull
.Where(x => x.ActiveTestMerges.Count == model.NewTestMerges.Count &&
x.ActiveTestMerges.Select(y => y.TestMerge)
.All(y => model.NewTestMerges.Any(z =>
y.Number == z.Number &&
y.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal) &&
(y.Comment?.Trim().ToUpperInvariant() == z.Comment?.Trim().ToUpperInvariant() || z.Comment == null))))
.FirstOrDefault();
if (revInfoWereLookingFor == null && model.NewTestMerges.Count > 1)
{
// okay try to add at least SOME prs we've seen before
var search = model.NewTestMerges.ToList();
var appliedTestMergeIds = new List <long>();
Models.RevisionInformation lastGoodRevInfo = null;
do
{
foreach (var I in search)
{
revInfoWereLookingFor = dbPull
.Where(x => model.NewTestMerges.Any(z =>
x.PrimaryTestMerge.Number == z.Number &&
x.PrimaryTestMerge.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal) &&
(x.PrimaryTestMerge.Comment?.Trim().ToUpperInvariant() == z.Comment?.Trim().ToUpperInvariant() || z.Comment == null)) &&
x.ActiveTestMerges.Select(y => y.TestMerge).All(y => appliedTestMergeIds.Contains(y.Id)))
.FirstOrDefault();
if (revInfoWereLookingFor != null)
{
lastGoodRevInfo = revInfoWereLookingFor;
appliedTestMergeIds.Add(revInfoWereLookingFor.PrimaryTestMerge.Id);
search.Remove(I);
break;
}
}
}while (revInfoWereLookingFor != null && search.Count > 0);
revInfoWereLookingFor = lastGoodRevInfo;
needToApplyRemainingPrs = search.Count != 0;
if (needToApplyRemainingPrs)
{
model.NewTestMerges = search;
}
}
else if (revInfoWereLookingFor != null)
{
needToApplyRemainingPrs = false;
}
}
}
if (revInfoWereLookingFor != null)
{
// goteem
await repo.ResetToSha(revInfoWereLookingFor.CommitSha, NextProgressReporter(), cancellationToken).ConfigureAwait(false);
lastRevisionInfo = revInfoWereLookingFor;
}
if (needToApplyRemainingPrs)
{
// an invocation of LoadRevisionInformation could have already loaded this user
var contextUser = databaseContext.Users.Local.Where(x => x.Id == AuthenticationContext.User.Id).FirstOrDefault();
if (contextUser == default)
{
contextUser = new Models.User
{
Id = AuthenticationContext.User.Id
};
databaseContext.Users.Attach(contextUser);
}
else
{
Logger.LogTrace("Skipping attaching the user to the database context as it is already loaded!");
}
foreach (var I in model.NewTestMerges)
{
Octokit.PullRequest pr = null;
string errorMessage = null;
if (lastRevisionInfo.ActiveTestMerges.Any(x => x.TestMerge.Number == I.Number.Value))
{
throw new JobException("Cannot test merge the same PR twice in one HEAD!");
}
try
{
// load from cache if possible
if (prMap == null || !prMap.TryGetValue(I.Number.Value, out pr))
{
pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number.Value).ConfigureAwait(false);
}
}
catch (Octokit.RateLimitExceededException)
{
// you look at your anonymous access and sigh
errorMessage = "P.R.E. RATE LIMITED";
}
catch (Octokit.AuthorizationException)
{
errorMessage = "P.R.E. BAD CREDENTIALS";
}
catch (Octokit.NotFoundException)
{
// you look at your shithub and sigh
errorMessage = "P.R.E. NOT FOUND";
}
// we want to take the earliest truth possible to prevent RCEs, if this fails AddTestMerge will set it
if (I.PullRequestRevision == null && pr != null)
{
I.PullRequestRevision = pr.Head.Sha;
}
var mergeResult = await repo.AddTestMerge(I, committerName, currentModel.CommitterEmail, currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct).ConfigureAwait(false);
if (!mergeResult.HasValue)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "Merge of PR #{0} at {1} conflicted!", I.Number, I.PullRequestRevision.Substring(0, 7)));
}
++doneSteps;
var revInfoUpdateTask = UpdateRevInfo();
var tm = new Models.TestMerge
{
Author = pr?.User.Login ?? errorMessage,
BodyAtMerge = pr?.Body ?? errorMessage ?? String.Empty,
MergedAt = DateTimeOffset.Now,
TitleAtMerge = pr?.Title ?? errorMessage ?? String.Empty,
Comment = I.Comment,
Number = I.Number,
MergedBy = contextUser,
PullRequestRevision = I.PullRequestRevision,
Url = pr?.HtmlUrl ?? errorMessage
};
await revInfoUpdateTask.ConfigureAwait(false);
lastRevisionInfo.PrimaryTestMerge = tm;
lastRevisionInfo.ActiveTestMerges.Add(new RevInfoTestMerge
{
TestMerge = tm
});
}
}
}
var currentHead = repo.Head;
if (startSha != currentHead || (postUpdateSha != null && postUpdateSha != currentHead))
{
await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), false, ct).ConfigureAwait(false);
await UpdateRevInfo().ConfigureAwait(false);
}
await databaseContext.Save(ct).ConfigureAwait(false);
}
catch
{
doneSteps = 0;
numSteps = 2;
// the stuff didn't make it into the db, forget what we've done and abort
await repo.CheckoutObject(startReference ?? startSha, NextProgressReporter(), default).ConfigureAwait(false);
if (startReference != null && repo.Head != startSha)
{
await repo.ResetToSha(startSha, NextProgressReporter(), default).ConfigureAwait(false);
}
else
{
progressReporter(100);
}
throw;
}
}
}, cancellationToken).ConfigureAwait(false);
api.ActiveJob = job.ToApi();
return(Accepted(api));
}
/// <inheritdoc />
public async Task <IGitHubClient> CreateClient()
{
var token = await GetAccessToken();
return(_gitHubClientFactory.CreateClient(token));
}
/// <inheritdoc />
public async Task CompileProcess(Job job, IDatabaseContext databaseContext, Action <int> progressReporter, CancellationToken cancellationToken)
{
#pragma warning disable IDE0016 // Use 'throw' expression
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
#pragma warning restore IDE0016 // Use 'throw' expression
if (databaseContext == null)
{
throw new ArgumentNullException(nameof(databaseContext));
}
if (progressReporter == null)
{
throw new ArgumentNullException(nameof(progressReporter));
}
var ddSettingsTask = databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == metadata.Id).Select(x => new DreamDaemonSettings
{
StartupTimeout = x.StartupTimeout,
}).FirstOrDefaultAsync(cancellationToken);
var compileJobsTask = databaseContext.CompileJobs
.Where(x => x.Job.Instance.Id == metadata.Id)
.OrderByDescending(x => x.Job.StoppedAt)
.Select(x => new Job
{
StoppedAt = x.Job.StoppedAt,
StartedAt = x.Job.StartedAt
})
.Take(10)
.ToListAsync(cancellationToken);
var dreamMakerSettings = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(cancellationToken).ConfigureAwait(false);
if (dreamMakerSettings == default)
{
throw new JobException("Missing DreamMakerSettings in DB!");
}
var ddSettings = await ddSettingsTask.ConfigureAwait(false);
if (ddSettings == default)
{
throw new JobException("Missing DreamDaemonSettings in DB!");
}
Task <RepositorySettings> repositorySettingsTask = null;
string repoOwner = null;
string repoName = null;
CompileJob compileJob;
RevisionInformation revInfo;
using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false))
{
if (repo == null)
{
throw new JobException("Missing Repository!");
}
if (repo.IsGitHubRepository)
{
repoOwner = repo.GitHubOwner;
repoName = repo.GitHubRepoName;
repositorySettingsTask = databaseContext.RepositorySettings.Where(x => x.InstanceId == metadata.Id).Select(x => new RepositorySettings
{
AccessToken = x.AccessToken,
ShowTestMergeCommitters = x.ShowTestMergeCommitters,
PushTestMergeCommits = x.PushTestMergeCommits
}).FirstOrDefaultAsync(cancellationToken);
}
var repoSha = repo.Head;
revInfo = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha && x.Instance.Id == metadata.Id).Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy).FirstOrDefaultAsync().ConfigureAwait(false);
if (revInfo == default)
{
revInfo = new RevisionInformation
{
CommitSha = repoSha,
OriginCommitSha = repoSha,
Instance = new Models.Instance
{
Id = metadata.Id
}
};
logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, repoSha);
databaseContext.Instances.Attach(revInfo.Instance);
}
TimeSpan?averageSpan = null;
var previousCompileJobs = await compileJobsTask.ConfigureAwait(false);
if (previousCompileJobs.Count != 0)
{
var totalSpan = TimeSpan.Zero;
foreach (var I in previousCompileJobs)
{
totalSpan += I.StoppedAt.Value - I.StartedAt.Value;
}
averageSpan = totalSpan / previousCompileJobs.Count;
}
compileJob = await dreamMaker.Compile(revInfo, dreamMakerSettings, ddSettings.StartupTimeout.Value, repo, progressReporter, averageSpan, cancellationToken).ConfigureAwait(false);
}
compileJob.Job = job;
databaseContext.CompileJobs.Add(compileJob); // will be saved by job context
job.PostComplete = ct => compileJobConsumer.LoadCompileJob(compileJob, ct);
if (repositorySettingsTask != null)
{
var repositorySettings = await repositorySettingsTask.ConfigureAwait(false);
if (repositorySettings == default)
{
throw new JobException("Missing repository settings!");
}
if (repositorySettings.AccessToken != null)
{
// potential for commenting on a test merge change
var outgoingCompileJob = LatestCompileJob();
if (outgoingCompileJob != null && outgoingCompileJob.RevisionInformation.CommitSha != compileJob.RevisionInformation.CommitSha && repositorySettings.PostTestMergeComment.Value)
{
var gitHubClient = gitHubClientFactory.CreateClient(repositorySettings.AccessToken);
async Task CommentOnPR(int prNumber, string comment)
{
try
{
await gitHubClient.Issue.Comment.Create(repoOwner, repoName, prNumber, comment).ConfigureAwait(false);
}
catch (ApiException e)
{
logger.LogWarning("Error posting GitHub comment! Exception: {0}", e);
}
}
var tasks = new List <Task>();
string FormatTestMerge(TestMerge testMerge, bool updated) => String.Format(CultureInfo.InvariantCulture, "#### Test Merge {4}{0}{0}##### Server Instance{0}{5}{1}{0}{0}##### Revision{0}Origin: {6}{0}Pull Request: {2}{0}Server: {7}{3}",
Environment.NewLine,
repositorySettings.ShowTestMergeCommitters.Value ? String.Format(CultureInfo.InvariantCulture, "{0}{0}##### Merged By{0}{1}", Environment.NewLine, testMerge.MergedBy.Name) : String.Empty,
testMerge.PullRequestRevision,
testMerge.Comment != null ? String.Format(CultureInfo.InvariantCulture, "{0}{0}##### Comment{0}{1}", Environment.NewLine, testMerge.Comment) : String.Empty,
updated ? "Updated" : "Deployed",
metadata.Name,
compileJob.RevisionInformation.OriginCommitSha,
compileJob.RevisionInformation.CommitSha);
// added prs
foreach (var I in compileJob
.RevisionInformation
.ActiveTestMerges
.Select(x => x.TestMerge)
.Where(x => !outgoingCompileJob
.RevisionInformation
.ActiveTestMerges
.Any(y => y.TestMerge.Number == x.Number)))
{
tasks.Add(CommentOnPR(I.Number.Value, FormatTestMerge(I, false)));
}
// removed prs
foreach (var I in outgoingCompileJob
.RevisionInformation
.ActiveTestMerges
.Select(x => x.TestMerge)
.Where(x => !compileJob
.RevisionInformation
.ActiveTestMerges
.Any(y => y.TestMerge.Number == x.Number)))
{
tasks.Add(CommentOnPR(I.Number.Value, "#### Test Merge Removed"));
}
// updated prs
foreach (var I in compileJob
.RevisionInformation
.ActiveTestMerges
.Select(x => x.TestMerge)
.Where(x => outgoingCompileJob
.RevisionInformation
.ActiveTestMerges
.Any(y => y.TestMerge.Number == x.Number)))
{
tasks.Add(CommentOnPR(I.Number.Value, FormatTestMerge(I, true)));
}
if (tasks.Any())
{
await Task.WhenAll(tasks).ConfigureAwait(false);
}
}
}
}
}
#pragma warning disable CA1502, CA1505 // TODO: Decomplexify
public async Task <IActionResult> Update([FromBody] Repository model, CancellationToken cancellationToken)
{
if (model == null)
{
throw new ArgumentNullException(nameof(model));
}
if (model.AccessUser == null ^ model.AccessToken == null)
{
return(BadRequest(new ErrorMessage(ErrorCode.RepoMismatchUserAndAccessToken)));
}
if (model.CheckoutSha != null && model.Reference != null)
{
return(BadRequest(new ErrorMessage(ErrorCode.RepoMismatchShaAndReference)));
}
if (model.CheckoutSha != null && model.UpdateFromOrigin == true)
{
return(BadRequest(new ErrorMessage(ErrorCode.RepoMismatchShaAndUpdate)));
}
if (model.NewTestMerges?.Any(x => model.NewTestMerges.Any(y => x != y && x.Number == y.Number)) == true)
{
return(BadRequest(new ErrorMessage(ErrorCode.RepoDuplicateTestMerge)));
}
if (model.CommitterName?.Length == 0)
{
return(BadRequest(new ErrorMessage(ErrorCode.RepoWhitespaceCommitterName)));
}
if (model.CommitterEmail?.Length == 0)
{
return(BadRequest(new ErrorMessage(ErrorCode.RepoWhitespaceCommitterEmail)));
}
var newTestMerges = model.NewTestMerges != null && model.NewTestMerges.Count > 0;
var userRights = (RepositoryRights)AuthenticationContext.GetRight(RightsType.Repository);
if (newTestMerges && !userRights.HasFlag(RepositoryRights.MergePullRequest))
{
return(Forbid());
}
var currentModel = await DatabaseContext
.RepositorySettings
.AsQueryable()
.Where(x => x.InstanceId == Instance.Id)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (currentModel == default)
{
return(Gone());
}
bool CheckModified <T>(Expression <Func <Api.Models.Internal.RepositorySettings, T> > expression, RepositoryRights requiredRight)
{
var memberSelectorExpression = (MemberExpression)expression.Body;
var property = (PropertyInfo)memberSelectorExpression.Member;
var newVal = property.GetValue(model);
if (newVal == null)
{
return(false);
}
if (!userRights.HasFlag(requiredRight) && property.GetValue(currentModel) != newVal)
{
return(true);
}
property.SetValue(currentModel, newVal);
return(false);
}
if (CheckModified(x => x.AccessToken, RepositoryRights.ChangeCredentials) ||
CheckModified(x => x.AccessUser, RepositoryRights.ChangeCredentials) ||
CheckModified(x => x.AutoUpdatesKeepTestMerges, RepositoryRights.ChangeAutoUpdateSettings) ||
CheckModified(x => x.AutoUpdatesSynchronize, RepositoryRights.ChangeAutoUpdateSettings) ||
CheckModified(x => x.CommitterEmail, RepositoryRights.ChangeCommitter) ||
CheckModified(x => x.CommitterName, RepositoryRights.ChangeCommitter) ||
CheckModified(x => x.PushTestMergeCommits, RepositoryRights.ChangeTestMergeCommits) ||
CheckModified(x => x.CreateGitHubDeployments, RepositoryRights.ChangeTestMergeCommits) ||
CheckModified(x => x.ShowTestMergeCommitters, RepositoryRights.ChangeTestMergeCommits) ||
CheckModified(x => x.PostTestMergeComment, RepositoryRights.ChangeTestMergeCommits) ||
(model.UpdateFromOrigin == true && !userRights.HasFlag(RepositoryRights.UpdateBranch)))
{
return(Forbid());
}
if (model.AccessToken?.Length == 0 && model.AccessUser?.Length == 0)
{
// setting an empty string clears everything
currentModel.AccessUser = null;
currentModel.AccessToken = null;
}
var canRead = userRights.HasFlag(RepositoryRights.Read);
var api = canRead ? currentModel.ToApi() : new Repository();
if (canRead)
{
var earlyOut = await WithComponentInstance(
async instance =>
{
var repoManager = instance.RepositoryManager;
if (repoManager.CloneInProgress)
{
return(Conflict(new ErrorMessage(ErrorCode.RepoCloning)));
}
if (repoManager.InUse)
{
return(Conflict(new ErrorMessage(ErrorCode.RepoBusy)));
}
using var repo = await repoManager.LoadRepository(cancellationToken).ConfigureAwait(false);
if (repo == null)
{
return(Conflict(new ErrorMessage(ErrorCode.RepoMissing)));
}
await PopulateApi(api, repo, DatabaseContext, Instance, cancellationToken).ConfigureAwait(false);
if (model.Origin != null && model.Origin != repo.Origin)
{
return(BadRequest(new ErrorMessage(ErrorCode.RepoCantChangeOrigin)));
}
return(null);
})
.ConfigureAwait(false);
if (earlyOut != null)
{
return(earlyOut);
}
}
// this is just db stuf so stow it away
await DatabaseContext.Save(cancellationToken).ConfigureAwait(false);
// format the job description
string description = null;
if (model.UpdateFromOrigin == true)
{
if (model.Reference != null)
{
description = String.Format(CultureInfo.InvariantCulture, "Fetch and hard reset repository to origin/{0}", model.Reference);
}
else if (model.CheckoutSha != null)
{
description = String.Format(CultureInfo.InvariantCulture, "Fetch and checkout {0} in repository", model.CheckoutSha);
}
else
{
description = "Pull current repository reference";
}
}
else if (model.Reference != null || model.CheckoutSha != null)
{
description = String.Format(CultureInfo.InvariantCulture, "Checkout repository {0} {1}", model.Reference != null ? "reference" : "SHA", model.Reference ?? model.CheckoutSha);
}
if (newTestMerges)
{
description = String.Format(CultureInfo.InvariantCulture, "{0}est merge(s) {1}{2}",
description != null ? String.Format(CultureInfo.InvariantCulture, "{0} and t", description) : "T",
String.Join(", ", model.NewTestMerges.Select(x =>
String.Format(CultureInfo.InvariantCulture, "#{0}{1}", x.Number,
x.TargetCommitSha != null ? String.Format(CultureInfo.InvariantCulture, " at {0}", x.TargetCommitSha.Substring(0, 7)) : String.Empty))),
description != null ? String.Empty : " in repository");
}
if (description == null)
{
return(Json(api)); // no git changes
}
async Task <IActionResult> UpdateCallbackThatDesperatelyNeedsRefactoring(
IInstanceCore instance,
IDatabaseContextFactory databaseContextFactory,
Action <int> progressReporter,
CancellationToken ct)
{
var repoManager = instance.RepositoryManager;
using var repo = await repoManager.LoadRepository(ct).ConfigureAwait(false);
if (repo == null)
{
throw new JobException(ErrorCode.RepoMissing);
}
var modelHasShaOrReference = model.CheckoutSha != null || model.Reference != null;
var startReference = repo.Reference;
var startSha = repo.Head;
string postUpdateSha = null;
if (newTestMerges && repo.RemoteGitProvider == RemoteGitProvider.Unknown)
{
throw new JobException(ErrorCode.RepoUnsupportedTestMergeRemote);
}
var committerName = currentModel.ShowTestMergeCommitters.Value
? AuthenticationContext.User.Name
: currentModel.CommitterName;
var hardResettingToOriginReference = model.UpdateFromOrigin == true && model.Reference != null;
var numSteps = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0) + (!modelHasShaOrReference ? 2 : (hardResettingToOriginReference ? 3 : 1));
var doneSteps = 0;
Action <int> NextProgressReporter()
{
var tmpDoneSteps = doneSteps;
++doneSteps;
return(progress => progressReporter((progress + (100 * tmpDoneSteps)) / numSteps));
}
progressReporter(0);
// get a base line for where we are
Models.RevisionInformation lastRevisionInfo = null;
var attachedInstance = new Models.Instance
{
Id = Instance.Id
};
Task CallLoadRevInfo(Models.TestMerge testMergeToAdd = null, string lastOriginCommitSha = null) => databaseContextFactory
.UseContext(
async databaseContext =>
{
databaseContext.Instances.Attach(attachedInstance);
var previousRevInfo = lastRevisionInfo;
var needsUpdate = await LoadRevisionInformation(
repo,
databaseContext,
attachedInstance,
lastOriginCommitSha,
x => lastRevisionInfo = x,
ct)
.ConfigureAwait(false);
if (testMergeToAdd != null)
{
// rev info may have already loaded the user
var mergedBy = databaseContext.Users.Local.FirstOrDefault(x => x.Id == AuthenticationContext.User.Id);
if (mergedBy == default)
{
mergedBy = new Models.User
{
Id = AuthenticationContext.User.Id
};
databaseContext.Users.Attach(mergedBy);
}
testMergeToAdd.MergedBy = mergedBy;
foreach (var activeTestMerge in previousRevInfo.ActiveTestMerges)
{
lastRevisionInfo.ActiveTestMerges.Add(activeTestMerge);
}
lastRevisionInfo.ActiveTestMerges.Add(new RevInfoTestMerge
{
TestMerge = testMergeToAdd
});
lastRevisionInfo.PrimaryTestMerge = testMergeToAdd;
needsUpdate = true;
}
if (needsUpdate)
{
await databaseContext.Save(cancellationToken).ConfigureAwait(false);
}
});
await CallLoadRevInfo().ConfigureAwait(false);
// apply new rev info, tracking applied test merges
Task UpdateRevInfo(Models.TestMerge testMergeToAdd = null) => CallLoadRevInfo(testMergeToAdd, lastRevisionInfo.OriginCommitSha);
try
{
// fetch/pull
if (model.UpdateFromOrigin == true)
{
if (!repo.Tracking)
{
throw new JobException(ErrorCode.RepoReferenceRequired);
}
await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct).ConfigureAwait(false);
doneSteps = 1;
if (!modelHasShaOrReference)
{
var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, NextProgressReporter(), ct).ConfigureAwait(false);
if (!fastForward.HasValue)
{
throw new JobException(ErrorCode.RepoMergeConflict);
}
lastRevisionInfo.OriginCommitSha = await repo.GetOriginSha(cancellationToken).ConfigureAwait(false);
await UpdateRevInfo().ConfigureAwait(false);
if (fastForward.Value)
{
await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false);
postUpdateSha = repo.Head;
}
else
{
NextProgressReporter()(100);
}
}
}
// checkout/hard reset
if (modelHasShaOrReference)
{
var validCheckoutSha =
model.CheckoutSha != null &&
!repo.Head.StartsWith(model.CheckoutSha, StringComparison.OrdinalIgnoreCase);
var validCheckoutReference =
model.Reference != null &&
!repo.Reference.Equals(model.Reference, StringComparison.OrdinalIgnoreCase);
if (validCheckoutSha || validCheckoutReference)
{
var committish = model.CheckoutSha ?? model.Reference;
var isSha = await repo.IsSha(committish, cancellationToken).ConfigureAwait(false);
if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null))
{
throw new JobException(ErrorCode.RepoSwappedShaOrReference);
}
await repo.CheckoutObject(committish, NextProgressReporter(), ct).ConfigureAwait(false);
await CallLoadRevInfo().ConfigureAwait(false); // we've either seen origin before or what we're checking out is on origin
}
else
{
NextProgressReporter()(100);
}
if (hardResettingToOriginReference)
{
if (!repo.Tracking)
{
throw new JobException(ErrorCode.RepoReferenceNotTracking);
}
await repo.ResetToOrigin(NextProgressReporter(), ct).ConfigureAwait(false);
await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false);
await CallLoadRevInfo().ConfigureAwait(false);
// repo head is on origin so force this
// will update the db if necessary
lastRevisionInfo.OriginCommitSha = repo.Head;
}
}
// test merging
if (newTestMerges)
{
if (repo.RemoteGitProvider == RemoteGitProvider.Unknown)
{
throw new JobException(ErrorCode.RepoTestMergeInvalidRemote);
}
// bit of sanitization
foreach (var I in model.NewTestMerges.Where(x => String.IsNullOrWhiteSpace(x.TargetCommitSha)))
{
I.TargetCommitSha = null;
}
var gitHubClient = currentModel.AccessToken != null
? gitHubClientFactory.CreateClient(currentModel.AccessToken)
: gitHubClientFactory.CreateClient();
var repoOwner = repo.RemoteRepositoryOwner;
var repoName = repo.RemoteRepositoryName;
// optimization: if we've already merged these exact same commits in this fashion before, just find the rev info for it and check it out
Models.RevisionInformation revInfoWereLookingFor = null;
bool needToApplyRemainingPrs = true;
if (lastRevisionInfo.OriginCommitSha == lastRevisionInfo.CommitSha)
{
bool cantSearch = false;
foreach (var I in model.NewTestMerges)
{
if (I.TargetCommitSha != null)
#pragma warning disable CA1308 // Normalize strings to uppercase
{
I.TargetCommitSha = I.TargetCommitSha?.ToLowerInvariant(); // ala libgit2
}
#pragma warning restore CA1308 // Normalize strings to uppercase
else
{
try
{
// retrieve the latest sha
var pr = await repo.GetTestMerge(I, currentModel, ct).ConfigureAwait(false);
// we want to take the earliest truth possible to prevent RCEs, if this fails AddTestMerge will set it
I.TargetCommitSha = pr.TargetCommitSha;
}
catch
{
cantSearch = true;
break;
}
}
}
if (!cantSearch)
{
List <Models.RevisionInformation> dbPull = null;
await databaseContextFactory.UseContext(
async databaseContext =>
dbPull = await databaseContext.RevisionInformations
.AsQueryable()
.Where(x => x.Instance.Id == Instance.Id &&
x.OriginCommitSha == lastRevisionInfo.OriginCommitSha &&
x.ActiveTestMerges.Count <= model.NewTestMerges.Count &&
x.ActiveTestMerges.Count > 0)
.Include(x => x.ActiveTestMerges)
.ThenInclude(x => x.TestMerge)
.ToListAsync(cancellationToken)
.ConfigureAwait(false))
.ConfigureAwait(false);
// split here cause this bit has to be done locally
revInfoWereLookingFor = dbPull
.Where(x => x.ActiveTestMerges.Count == model.NewTestMerges.Count &&
x.ActiveTestMerges.Select(y => y.TestMerge)
.All(y => model.NewTestMerges.Any(z =>
y.Number == z.Number &&
y.TargetCommitSha.StartsWith(z.TargetCommitSha, StringComparison.Ordinal) &&
(y.Comment?.Trim().ToUpperInvariant() == z.Comment?.Trim().ToUpperInvariant() || z.Comment == null))))
.FirstOrDefault();
if (revInfoWereLookingFor == default && model.NewTestMerges.Count > 1)
{
// okay try to add at least SOME prs we've seen before
var search = model.NewTestMerges.ToList();
var appliedTestMergeIds = new List <long>();
Models.RevisionInformation lastGoodRevInfo = null;
do
{
foreach (var I in search)
{
revInfoWereLookingFor = dbPull
.Where(testRevInfo =>
{
if (testRevInfo.PrimaryTestMerge == null)
{
return(false);
}
var testMergeMatch = model.NewTestMerges.Any(testTestMerge =>
{
var numberMatch = testRevInfo.PrimaryTestMerge.Number == testTestMerge.Number;
if (!numberMatch)
{
return(false);
}
var shaMatch = testRevInfo.PrimaryTestMerge.TargetCommitSha.StartsWith(
testTestMerge.TargetCommitSha,
StringComparison.Ordinal);
if (!shaMatch)
{
return(false);
}
var commentMatch = testRevInfo.PrimaryTestMerge.Comment == testTestMerge.Comment;
return(commentMatch);
});
if (!testMergeMatch)
{
return(false);
}
var previousTestMergesMatch = testRevInfo
.ActiveTestMerges
.Select(previousRevInfoTestMerge => previousRevInfoTestMerge.TestMerge)
.All(previousTestMerge => appliedTestMergeIds.Contains(previousTestMerge.Id));
return(previousTestMergesMatch);
})
.FirstOrDefault();
if (revInfoWereLookingFor != null)
{
lastGoodRevInfo = revInfoWereLookingFor;
appliedTestMergeIds.Add(revInfoWereLookingFor.PrimaryTestMerge.Id);
search.Remove(I);
break;
}
}
}while (revInfoWereLookingFor != null && search.Count > 0);
revInfoWereLookingFor = lastGoodRevInfo;
needToApplyRemainingPrs = search.Count != 0;
if (needToApplyRemainingPrs)
{
model.NewTestMerges = search;
}
}
else if (revInfoWereLookingFor != null)
{
needToApplyRemainingPrs = false;
}
}
}
if (revInfoWereLookingFor != null)
{
// goteem
Logger.LogDebug("Reusing existing SHA {0}...", revInfoWereLookingFor.CommitSha);
await repo.ResetToSha(revInfoWereLookingFor.CommitSha, NextProgressReporter(), cancellationToken).ConfigureAwait(false);
lastRevisionInfo = revInfoWereLookingFor;
}
if (needToApplyRemainingPrs)
{
foreach (var I in model.NewTestMerges)
{
if (lastRevisionInfo.ActiveTestMerges.Any(x => x.TestMerge.Number == I.Number))
{
throw new JobException(ErrorCode.RepoDuplicateTestMerge);
}
var fullTestMergeTask = repo.GetTestMerge(I, currentModel, ct);
var mergeResult = await repo.AddTestMerge(
I,
committerName,
currentModel.CommitterEmail,
currentModel.AccessUser,
currentModel.AccessToken,
NextProgressReporter(),
ct).ConfigureAwait(false);
if (mergeResult == null)
{
throw new JobException(
ErrorCode.RepoTestMergeConflict,
new JobException(
$"Test Merge #{I.Number} at {I.TargetCommitSha.Substring(0, 7)} conflicted!"));
}
Models.TestMerge fullTestMerge;
try
{
fullTestMerge = await fullTestMergeTask.ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning("Error retrieving metadata for test merge #{0}!", I.Number);
fullTestMerge = new Models.TestMerge
{
Author = ex.Message,
BodyAtMerge = ex.Message,
MergedAt = DateTimeOffset.UtcNow,
TitleAtMerge = ex.Message,
Comment = I.Comment,
Number = I.Number,
Url = ex.Message
};
}
// Ensure we're getting the full sha from git itself
fullTestMerge.TargetCommitSha = I.TargetCommitSha;
// MergedBy will be set later
++doneSteps;
await UpdateRevInfo(fullTestMerge).ConfigureAwait(false);
}
}
}
var currentHead = repo.Head;
if (startSha != currentHead || (postUpdateSha != null && postUpdateSha != currentHead))
{
await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), false, ct).ConfigureAwait(false);
await UpdateRevInfo().ConfigureAwait(false);
}
return(null);
}
catch
{
doneSteps = 0;
numSteps = 2;
// Forget what we've done and abort
// DCTx2: Cancellation token is for job, operations should always run
await repo.CheckoutObject(startReference ?? startSha, NextProgressReporter(), default).ConfigureAwait(false);
if (startReference != null && repo.Head != startSha)
{
await repo.ResetToSha(startSha, NextProgressReporter(), default).ConfigureAwait(false);
}
else
{
progressReporter(100);
}
throw;
}
}
var job = new Models.Job
{
Description = description,
StartedBy = AuthenticationContext.User,
Instance = Instance,
CancelRightsType = RightsType.Repository,
CancelRight = (ulong)RepositoryRights.CancelPendingChanges,
};
// Time to access git, do it in a job
await jobManager.RegisterOperation(
job,
(core, databaseContextFactory, paramJob, progressReporter, ct) =>
UpdateCallbackThatDesperatelyNeedsRefactoring(
core,
databaseContextFactory,
progressReporter,
ct),
cancellationToken)
.ConfigureAwait(false);
api.ActiveJob = job.ToApi();
return(Accepted(api));
}