/// <inheritdoc />
public async Task ChangeVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken)
{
if (version == null)
{
throw new ArgumentNullException(nameof(version));
}
var versionKey = await InstallVersion(version, customVersionStream, cancellationToken).ConfigureAwait(false);
using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
{
await ioManager.WriteAllBytes(ActiveVersionFileName, Encoding.UTF8.GetBytes(versionKey), cancellationToken).ConfigureAwait(false);
await eventConsumer.HandleEvent(
EventType.ByondActiveVersionChange,
new List <string>
{
ActiveVersion != null
? VersionKey(ActiveVersion, true)
: null,
versionKey,
},
cancellationToken)
.ConfigureAwait(false);
// We reparse the version key because it could be changed after a custom install.
ActiveVersion = Version.Parse(versionKey);
}
}
private async Task Consume()
{
var token = _cancellationToken.Token;
var serializerSettings = JsonConverterExtensions.CreateSettings();
while (!token.IsCancellationRequested)
{
if (_consumer.Consume(out var message, 1000))
{
var sw = Stopwatch.StartNew();
try
{
dynamic evnt = JsonConvert.DeserializeObject(message.Value, serializerSettings);
if (evnt as IEvent == null)
{
continue;
}
evnt.Enrich(message.ToEventMetadata());
await _handler.HandleEvent(evnt);
_logger.Debug($"Processed event {evnt.GetType().FullName} in {sw.ElapsedMilliseconds} ms.");
}
catch (JsonSerializationException e)
{
_logger.Error(e, "Could not deserialize event!");
}
catch (Exception e)
{
_logger.Error(e, "Could not process event!");
}
}
}
}
/// <summary>
/// Pull the repository and compile for every set of given <paramref name="minutes"/>
/// </summary>
/// <param name="minutes">How many minutes the operation should repeat. Does not include running time</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
#pragma warning disable CA1502 // TODO: Decomplexify
async Task TimerLoop(uint minutes, CancellationToken cancellationToken)
{
while (true)
{
try
{
await Task.Delay(TimeSpan.FromMinutes(minutes > Int32.MaxValue ? Int32.MaxValue : (int)minutes), cancellationToken).ConfigureAwait(false);
logger.LogInformation("Beginning auto update...");
await eventConsumer.HandleEvent(EventType.InstanceAutoUpdateStart, new List <string>(), cancellationToken).ConfigureAwait(false);
try
{
Models.User user = null;
await databaseContextFactory.UseContext(async (db) => user = await db.Users.Where(x => x.CanonicalName == Api.Models.User.AdminName.ToUpperInvariant()).FirstAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false);
var repositoryUpdateJob = new Job
{
Instance = new Models.Instance
{
Id = metadata.Id
},
Description = "Scheduled repository update",
CancelRightsType = RightsType.Repository,
CancelRight = (ulong)RepositoryRights.CancelPendingChanges,
StartedBy = user
};
string deploySha = null;
await jobManager.RegisterOperation(repositoryUpdateJob, async (paramJob, databaseContext, progressReporter, jobCancellationToken) =>
{
var repositorySettingsTask = databaseContext.RepositorySettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(jobCancellationToken);
// assume 5 steps with synchronize
const int ProgressSections = 7;
const int ProgressStep = 100 / ProgressSections;
const int NumSteps = 3;
var doneSteps = 0;
Action <int> NextProgressReporter()
{
var tmpDoneSteps = doneSteps;
++doneSteps;
return(progress => progressReporter((progress + (100 * tmpDoneSteps)) / NumSteps));
}
using (var repo = await RepositoryManager.LoadRepository(jobCancellationToken).ConfigureAwait(false))
{
if (repo == null)
{
logger.LogTrace("Aborting repo update, no repository!");
return;
}
var startSha = repo.Head;
if (!repo.Tracking)
{
logger.LogTrace("Aborting repo update, active ref not tracking any remote branch!");
deploySha = startSha;
return;
}
var repositorySettings = await repositorySettingsTask.ConfigureAwait(false);
// the main point of auto update is to pull the remote
await repo.FetchOrigin(repositorySettings.AccessUser, repositorySettings.AccessToken, NextProgressReporter(), jobCancellationToken).ConfigureAwait(false);
RevisionInformation currentRevInfo = null;
bool hasDbChanges = false;
Task <RevisionInformation> LoadRevInfo() => databaseContext.RevisionInformations
.Where(x => x.CommitSha == startSha && x.Instance.Id == metadata.Id)
.Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge)
.FirstOrDefaultAsync(cancellationToken);
async Task UpdateRevInfo(string currentHead, bool onOrigin)
{
if (currentRevInfo == null)
{
currentRevInfo = await LoadRevInfo().ConfigureAwait(false);
}
if (currentRevInfo == default)
{
logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, currentHead);
onOrigin = true;
}
var attachedInstance = new Models.Instance
{
Id = metadata.Id
};
var oldRevInfo = currentRevInfo;
currentRevInfo = new RevisionInformation
{
CommitSha = currentHead,
OriginCommitSha = onOrigin ? currentHead : oldRevInfo.OriginCommitSha,
Instance = attachedInstance
};
if (!onOrigin)
{
currentRevInfo.ActiveTestMerges = new List <RevInfoTestMerge>(oldRevInfo.ActiveTestMerges);
}
databaseContext.Instances.Attach(attachedInstance);
databaseContext.RevisionInformations.Add(currentRevInfo);
hasDbChanges = true;
}
// take appropriate auto update actions
bool shouldSyncTracked;
if (repositorySettings.AutoUpdatesKeepTestMerges.Value)
{
logger.LogTrace("Preserving test merges...");
var currentRevInfoTask = LoadRevInfo();
var result = await repo.MergeOrigin(repositorySettings.CommitterName, repositorySettings.CommitterEmail, NextProgressReporter(), jobCancellationToken).ConfigureAwait(false);
if (!result.HasValue)
{
throw new JobException("Merge conflict while preserving test merges!");
}
currentRevInfo = await currentRevInfoTask.ConfigureAwait(false);
var lastRevInfoWasOriginCommit = currentRevInfo == default || currentRevInfo.CommitSha == currentRevInfo.OriginCommitSha;
var stillOnOrigin = result.Value && lastRevInfoWasOriginCommit;
var currentHead = repo.Head;
if (currentHead != startSha)
{
await UpdateRevInfo(currentHead, stillOnOrigin).ConfigureAwait(false);
shouldSyncTracked = stillOnOrigin;
}
else
{
shouldSyncTracked = false;
}
}
else
{
logger.LogTrace("Not preserving test merges...");
await repo.ResetToOrigin(NextProgressReporter(), jobCancellationToken).ConfigureAwait(false);
var currentHead = repo.Head;
currentRevInfo = await databaseContext.RevisionInformations
.Where(x => x.CommitSha == currentHead && x.Instance.Id == metadata.Id)
.FirstOrDefaultAsync(jobCancellationToken).ConfigureAwait(false);
if (currentHead != startSha && currentRevInfo != default)
{
await UpdateRevInfo(currentHead, true).ConfigureAwait(false);
}
shouldSyncTracked = true;
}
// synch if necessary
if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head)
{
var pushedOrigin = await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, repositorySettings.CommitterName, repositorySettings.CommitterEmail, NextProgressReporter(), shouldSyncTracked, jobCancellationToken).ConfigureAwait(false);
var currentHead = repo.Head;
if (currentHead != currentRevInfo.CommitSha)
{
await UpdateRevInfo(currentHead, pushedOrigin).ConfigureAwait(false);
}
}
if (hasDbChanges)
{
try
{
await databaseContext.Save(cancellationToken).ConfigureAwait(false);
}
catch
{
await repo.ResetToSha(startSha, progressReporter, default).ConfigureAwait(false);
throw;
}
}
progressReporter(5 * ProgressStep);
deploySha = repo.Head;
}
}, cancellationToken).ConfigureAwait(false);
await jobManager.WaitForJobCompletion(repositoryUpdateJob, user, cancellationToken, default).ConfigureAwait(false);
if (deploySha == null)
{
logger.LogTrace("Aborting auto update, repository error!");
continue;
}
if (deploySha == LatestCompileJob()?.RevisionInformation.CommitSha)
{
logger.LogTrace("Aborting auto update, same revision as latest CompileJob");
continue;
}
// finally set up the job
var compileProcessJob = new Job
{
StartedBy = user,
Instance = repositoryUpdateJob.Instance,
Description = "Scheduled code deployment",
CancelRightsType = RightsType.DreamMaker,
CancelRight = (ulong)DreamMakerRights.CancelCompile
};
await jobManager.RegisterOperation(compileProcessJob, CompileProcess, cancellationToken).ConfigureAwait(false);
await jobManager.WaitForJobCompletion(compileProcessJob, user, cancellationToken, default).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
logger.LogDebug("Cancelled auto update job!");
throw;
}
catch (Exception e)
{
logger.LogWarning("Error in auto update loop! Exception: {0}", e);
continue;
}
}
catch (OperationCanceledException)
{
break;
}
}
logger.LogTrace("Leaving auto update loop...");
}
/// <summary>
/// Executes and populate a given <paramref name="job"/>
/// </summary>
/// <param name="job">The <see cref="Models.CompileJob"/> to run and populate</param>
/// <param name="dreamMakerSettings">The <see cref="Api.Models.DreamMaker"/> settings to use</param>
/// <param name="byondLock">The <see cref="IByondExecutableLock"/> to use</param>
/// <param name="repository">The <see cref="IRepository"/> to use</param>
/// <param name="apiValidateTimeout">The timeout for validating the DMAPI</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task RunCompileJob(Models.CompileJob job, Api.Models.DreamMaker dreamMakerSettings, IByondExecutableLock byondLock, IRepository repository, uint apiValidateTimeout, CancellationToken cancellationToken)
{
var jobPath = job.DirectoryName.ToString();
logger.LogTrace("Compile output GUID: {0}", jobPath);
try
{
var dirA = ioManager.ConcatPath(jobPath, ADirectoryName);
var dirB = ioManager.ConcatPath(jobPath, BDirectoryName);
// copy the repository
logger.LogTrace("Copying repository to game directory...");
var resolvedADirectory = ioManager.ResolvePath(dirA);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(resolvedADirectory, cancellationToken).ConfigureAwait(false);
// repository closed now
// run precompile scripts
await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> {
resolvedADirectory, repoOrigin
}, cancellationToken).ConfigureAwait(false);
// determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var foundPaths = await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false);
var foundPath = foundPaths.FirstOrDefault();
if (foundPath == default)
{
throw new JobException("Unable to find any .dme!");
}
var dmeWithExtension = ioManager.GetFileName(foundPath);
job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1);
}
else
{
var targetDme = ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension));
var targetDmeExists = await ioManager.FileExists(targetDme, cancellationToken).ConfigureAwait(false);
if (!targetDmeExists)
{
throw new JobException("Unable to locate specified .dme!");
}
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
// run compiler
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
// verify api
try
{
if (exitCode != 0)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output));
}
await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false);
}
catch (JobException)
{
// DD never validated or compile failed
await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> {
resolvedADirectory, exitCode == 0 ? "1" : "0"
}, cancellationToken).ConfigureAwait(false);
throw;
}
logger.LogTrace("Running post compile event...");
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
resolvedADirectory
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Duplicating compiled game...");
// duplicate the dmb et al
await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
// symlink in the static data
var symATask = configuration.SymlinkStaticFilesTo(resolvedADirectory, cancellationToken);
var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken);
await Task.WhenAll(symATask, symBTask).ConfigureAwait(false);
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
}
catch (Exception e)
{
await CleanupFailedCompile(job, e is OperationCanceledException, cancellationToken).ConfigureAwait(false);
throw;
}
}
/// <inheritdoc />
public async Task <Models.CompileJob> Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action <int> progressReporter, TimeSpan?estimatedDuration, CancellationToken cancellationToken)
{
if (revisionInformation == null)
{
throw new ArgumentNullException(nameof(revisionInformation));
}
if (dreamMakerSettings == null)
{
throw new ArgumentNullException(nameof(dreamMakerSettings));
}
if (repository == null)
{
throw new ArgumentNullException(nameof(repository));
}
if (progressReporter == null)
{
throw new ArgumentNullException(nameof(progressReporter));
}
if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe)
{
throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!");
}
logger.LogTrace("Begin Compile");
var job = new Models.CompileJob
{
DirectoryName = Guid.NewGuid(),
DmeName = dreamMakerSettings.ProjectName,
RevisionInformation = revisionInformation
};
logger.LogTrace("Compile output GUID: {0}", job.DirectoryName);
lock (this)
{
if (compiling)
{
throw new JobException("There is already a compile job in progress!");
}
compiling = true;
}
using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
async Task ProgressTask()
{
if (!estimatedDuration.HasValue)
{
return;
}
progressReporter(0);
var ct = progressCts.Token;
var sleepInterval = estimatedDuration.Value / 100;
try
{
for (var I = 0; I < 99; ++I)
{
await Task.Delay(sleepInterval, progressCts.Token).ConfigureAwait(false);
progressReporter(I + 1);
}
}
catch (OperationCanceledException) { }
}
var progressTask = ProgressTask();
try
{
var commitInsert = revisionInformation.CommitSha.Substring(0, 7);
string remoteCommitInsert;
if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha)
{
commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert);
remoteCommitInsert = String.Empty;
}
else
{
remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7));
}
var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})",
String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x =>
{
var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7));
if (x.Comment != null)
{
result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment);
}
return(result);
})));
using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false))
{
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false);
async Task CleanupFailedCompile(bool cancelled)
{
logger.LogTrace("Cleaning compile directory...");
var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken);
try
{
await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e);
}
await chatTask.ConfigureAwait(false);
};
try
{
await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false);
var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName);
logger.LogTrace("Copying repository to game directory...");
//copy the repository
var fullDirA = ioManager.ResolvePath(dirA);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false);
//run precompile scripts
var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName));
await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> {
resolvedGameDirectory, repoOrigin
}, cancellationToken).ConfigureAwait(false);
//determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault();
if (path == default)
{
throw new JobException("Unable to find any .dme!");
}
var dmeWithExtension = ioManager.GetFileName(path);
job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1);
}
else if (!await ioManager.FileExists(ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)), cancellationToken).ConfigureAwait(false))
{
throw new JobException("Unable to locate specified .dme!");
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
//run compiler, verify api
job.ByondVersion = byondLock.Version.ToString();
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
try
{
if (exitCode != 0)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output));
}
await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false);
}
catch (JobException)
{
//server never validated or compile failed
await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> {
resolvedGameDirectory, exitCode == 0 ? "1" : "0"
}, cancellationToken).ConfigureAwait(false);
throw;
}
logger.LogTrace("Running post compile event...");
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName))
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Duplicating compiled game...");
//duplicate the dmb et al
await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
//symlink in the static data
var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken);
var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken);
await Task.WhenAll(symATask, symBTask).ConfigureAwait(false);
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
return(job);
}
catch (Exception e)
{
await CleanupFailedCompile(e is OperationCanceledException).ConfigureAwait(false);
throw;
}
}
}
catch (OperationCanceledException)
{
await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false);
throw;
}
finally
{
compiling = false;
progressCts.Cancel();
await progressTask.ConfigureAwait(false);
}
}
}
/// <inheritdoc />
#pragma warning disable CA1506 // TODO: Decomplexify
public async Task <bool?> AddTestMerge(TestMergeParameters testMergeParameters, string committerName, string committerEmail, string username, string password, Action <int> progressReporter, CancellationToken cancellationToken)
{
if (testMergeParameters == null)
{
throw new ArgumentNullException(nameof(testMergeParameters));
}
if (committerName == null)
{
throw new ArgumentNullException(nameof(committerName));
}
if (committerEmail == null)
{
throw new ArgumentNullException(nameof(committerEmail));
}
if (progressReporter == null)
{
throw new ArgumentNullException(nameof(progressReporter));
}
logger.LogDebug("Begin AddTestMerge: #{0} at {1} ({4}) by <{2} ({3})>", testMergeParameters.Number, testMergeParameters.PullRequestRevision?.Substring(0, 7), committerName, committerEmail, testMergeParameters.Comment);
if (!IsGitHubRepository)
{
throw new InvalidOperationException("Test merging is only available on GitHub hosted origin repositories!");
}
var commitMessage = String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", testMergeParameters.Number.Value, testMergeParameters.Comment != null ? Environment.NewLine : String.Empty, testMergeParameters.Comment ?? String.Empty);
var prBranchName = String.Format(CultureInfo.InvariantCulture, "pr-{0}", testMergeParameters.Number);
var localBranchName = String.Format(CultureInfo.InvariantCulture, "pull/{0}/headrefs/heads/{1}", testMergeParameters.Number, prBranchName);
var refSpec = String.Format(CultureInfo.InvariantCulture, "pull/{0}/head:{1}", testMergeParameters.Number, prBranchName);
var refSpecList = new List <string> {
refSpec
};
var logMessage = String.Format(CultureInfo.InvariantCulture, "Merge remote pull request #{0}", testMergeParameters.Number);
var originalCommit = repository.Head;
MergeResult result = null;
var sig = new Signature(new Identity(committerName, committerEmail), DateTimeOffset.Now);
await Task.Factory.StartNew(() =>
{
try
{
try
{
logger.LogTrace("Fetching refspec {0}...", refSpec);
var remote = repository.Network.Remotes.First();
progressReporter(0);
Commands.Fetch((LibGit2Sharp.Repository)repository, remote.Name, refSpecList, new FetchOptions
{
Prune = true,
OnProgress = (a) => !cancellationToken.IsCancellationRequested,
OnTransferProgress = (a) =>
{
var percentage = 50 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2));
progressReporter((int)percentage);
return(!cancellationToken.IsCancellationRequested);
},
OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
CredentialsProvider = credentialsProvider.GenerateHandler(username, password)
}, logMessage);
}
catch (UserCancelledException) { }
cancellationToken.ThrowIfCancellationRequested();
repository.RemoveUntrackedFiles();
cancellationToken.ThrowIfCancellationRequested();
testMergeParameters.PullRequestRevision = repository.Lookup(testMergeParameters.PullRequestRevision ?? localBranchName).Sha;
cancellationToken.ThrowIfCancellationRequested();
logger.LogTrace("Merging {0} into {1}...", testMergeParameters.PullRequestRevision.Substring(0, 7), Reference);
result = repository.Merge(testMergeParameters.PullRequestRevision, sig, new MergeOptions
{
CommitOnSuccess = commitMessage == null,
FailOnConflict = true,
FastForwardStrategy = FastForwardStrategy.NoFastForward,
SkipReuc = true,
OnCheckoutProgress = (a, completedSteps, totalSteps) => progressReporter(50 + ((int)(((float)completedSteps) / totalSteps * 50)))
});
}
finally
{
repository.Branches.Remove(localBranchName);
}
cancellationToken.ThrowIfCancellationRequested();
if (result.Status == MergeStatus.Conflicts)
{
var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha;
logger.LogDebug("Merge conflict, aborting and reverting to {0}", revertTo);
RawCheckout(revertTo, progressReporter, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
}
repository.RemoveUntrackedFiles();
}, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false);
if (result.Status == MergeStatus.Conflicts)
{
await eventConsumer.HandleEvent(EventType.RepoMergeConflict, new List <string> {
originalCommit.Tip.Sha, testMergeParameters.PullRequestRevision, originalCommit.FriendlyName ?? UnknownReference, prBranchName
}, cancellationToken).ConfigureAwait(false);
return(null);
}
if (commitMessage != null && result.Status != MergeStatus.UpToDate)
{
logger.LogTrace("Committing merge: \"{0}\"...", commitMessage);
await Task.Factory.StartNew(() => repository.Commit(commitMessage, sig, sig, new CommitOptions
{
PrettifyMessage = true
}), cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false);
}
await eventConsumer.HandleEvent(EventType.RepoMergePullRequest, new List <string> {
testMergeParameters.Number.ToString(), testMergeParameters.PullRequestRevision, testMergeParameters.Comment
}, cancellationToken).ConfigureAwait(false);
return(result.Status != MergeStatus.NonFastForward);
}
/// <summary>
/// Installs a BYOND <paramref name="version"/> if it isn't already
/// </summary>
/// <param name="version">The BYOND <see cref="Version"/> to install</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task InstallVersion(Version version, CancellationToken cancellationToken)
{
var ourTcs = new TaskCompletionSource <object>();
Task inProgressTask;
var versionKey = VersionKey(version);
bool installed;
lock (installedVersions)
{
installed = installedVersions.TryGetValue(versionKey, out inProgressTask);
if (!installed)
{
installedVersions.Add(versionKey, ourTcs.Task);
}
}
if (installed)
{
using (cancellationToken.Register(() => ourTcs.SetCanceled()))
{
await Task.WhenAny(ourTcs.Task, inProgressTask).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
return;
}
}
else
{
logger.LogDebug("Requested BYOND version {0} not currently installed. Doing so now...");
}
// okay up to us to install it then
try
{
await eventConsumer.HandleEvent(EventType.ByondInstallStart, new List <string> {
versionKey
}, cancellationToken).ConfigureAwait(false);
var downloadTask = byondInstaller.DownloadVersion(version, cancellationToken);
await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false);
try
{
var download = await downloadTask.ConfigureAwait(false);
await ioManager.CreateDirectory(versionKey, cancellationToken).ConfigureAwait(false);
var extractPath = ioManager.ResolvePath(versionKey);
logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath);
await ioManager.ZipToDirectory(extractPath, download, cancellationToken).ConfigureAwait(false);
await byondInstaller.InstallByond(extractPath, version, cancellationToken).ConfigureAwait(false);
// make sure to do this last because this is what tells us we have a valid version in the future
await ioManager.WriteAllBytes(ioManager.ConcatPath(versionKey, VersionFileName), Encoding.UTF8.GetBytes(version.ToString()), cancellationToken).ConfigureAwait(false);
}
catch (WebException e)
{
// since the user can easily provide non-exitent version numbers, we'll turn this into a JobException
throw new JobException(ErrorCode.ByondDownloadFail, e);
}
catch (OperationCanceledException)
{
throw;
}
catch
{
await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false);
throw;
}
ourTcs.SetResult(null);
}
catch (Exception e)
{
if (!(e is OperationCanceledException))
{
await eventConsumer.HandleEvent(EventType.ByondInstallFail, new List <string> {
e.Message
}, cancellationToken).ConfigureAwait(false);
}
lock (installedVersions)
installedVersions.Remove(versionKey);
ourTcs.SetException(e);
throw;
}
}
private static void PublishToConsumer <TEventMessage>(IEventConsumer <TEventMessage> x, TEventMessage eventMessage)
{
x.HandleEvent(eventMessage);
}
/// <inheritdoc />
#pragma warning disable CA1506 // TODO: Decomplexify
public async Task <bool?> AddTestMerge(
TestMergeParameters testMergeParameters,
string committerName,
string committerEmail,
string username,
string password,
Action <int> progressReporter,
CancellationToken cancellationToken)
{
if (testMergeParameters == null)
{
throw new ArgumentNullException(nameof(testMergeParameters));
}
if (committerName == null)
{
throw new ArgumentNullException(nameof(committerName));
}
if (committerEmail == null)
{
throw new ArgumentNullException(nameof(committerEmail));
}
if (progressReporter == null)
{
throw new ArgumentNullException(nameof(progressReporter));
}
logger.LogDebug(
"Begin AddTestMerge: #{0} at {1} ({2}) by <{3} ({4})>",
testMergeParameters.Number,
testMergeParameters.TargetCommitSha?.Substring(0, 7),
testMergeParameters.Comment,
committerName,
committerEmail);
if (RemoteGitProvider == Api.Models.RemoteGitProvider.Unknown)
{
throw new InvalidOperationException("Cannot test merge with an Unknown RemoteGitProvider!");
}
var commitMessage = String.Format(
CultureInfo.InvariantCulture,
"TGS Test merge #{0}{1}{2}",
testMergeParameters.Number,
testMergeParameters.Comment != null
? Environment.NewLine
: String.Empty,
testMergeParameters.Comment ?? String.Empty);
var testMergeBranchName = String.Format(CultureInfo.InvariantCulture, "tm-{0}", testMergeParameters.Number);
var localBranchName = String.Format(CultureInfo.InvariantCulture, gitRemoteFeatures.TestMergeLocalBranchNameFormatter, testMergeParameters.Number, testMergeBranchName);
var refSpec = String.Format(CultureInfo.InvariantCulture, gitRemoteFeatures.TestMergeRefSpecFormatter, testMergeParameters.Number, testMergeBranchName);
var refSpecList = new List <string> {
refSpec
};
var logMessage = String.Format(CultureInfo.InvariantCulture, "Test merge #{0}", testMergeParameters.Number);
var originalCommit = libGitRepo.Head;
MergeResult result = null;
var sig = new Signature(new Identity(committerName, committerEmail), DateTimeOffset.UtcNow);
await Task.Factory.StartNew(
() =>
{
try
{
try
{
logger.LogTrace("Fetching refspec {0}...", refSpec);
var remote = libGitRepo.Network.Remotes.First();
progressReporter(0);
commands.Fetch(
libGitRepo,
refSpecList,
remote,
new FetchOptions
{
Prune = true,
OnProgress = (a) => !cancellationToken.IsCancellationRequested,
OnTransferProgress = (a) =>
{
var percentage = 50 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2));
progressReporter((int)percentage);
return(!cancellationToken.IsCancellationRequested);
},
OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
},
logMessage);
}
catch (UserCancelledException)
{
}
catch (LibGit2SharpException ex)
{
CheckBadCredentialsException(ex);
}
cancellationToken.ThrowIfCancellationRequested();
libGitRepo.RemoveUntrackedFiles();
cancellationToken.ThrowIfCancellationRequested();
testMergeParameters.TargetCommitSha = libGitRepo.Lookup(testMergeParameters.TargetCommitSha ?? localBranchName).Sha;
cancellationToken.ThrowIfCancellationRequested();
logger.LogTrace("Merging {0} into {1}...", testMergeParameters.TargetCommitSha.Substring(0, 7), Reference);
result = libGitRepo.Merge(testMergeParameters.TargetCommitSha, sig, new MergeOptions
{
CommitOnSuccess = commitMessage == null,
FailOnConflict = true,
FastForwardStrategy = FastForwardStrategy.NoFastForward,
SkipReuc = true,
OnCheckoutProgress = (a, completedSteps, totalSteps) => progressReporter(50 + ((int)(((float)completedSteps) / totalSteps * 50))),
});
}
finally
{
libGitRepo.Branches.Remove(localBranchName);
}
cancellationToken.ThrowIfCancellationRequested();
if (result.Status == MergeStatus.Conflicts)
{
var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha;
logger.LogDebug("Merge conflict, aborting and reverting to {0}", revertTo);
RawCheckout(revertTo, progressReporter, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
}
libGitRepo.RemoveUntrackedFiles();
},
cancellationToken,
DefaultIOManager.BlockingTaskCreationOptions,
TaskScheduler.Current)
.ConfigureAwait(false);
if (result.Status == MergeStatus.Conflicts)
{
await eventConsumer.HandleEvent(
EventType.RepoMergeConflict,
new List <string>
{
originalCommit.Tip.Sha,
testMergeParameters.TargetCommitSha,
originalCommit.FriendlyName ?? UnknownReference,
testMergeBranchName,
},
cancellationToken)
.ConfigureAwait(false);
return(null);
}
if (commitMessage != null && result.Status != MergeStatus.UpToDate)
{
logger.LogTrace("Committing merge: \"{0}\"...", commitMessage);
await Task.Factory.StartNew(
() => libGitRepo.Commit(commitMessage, sig, sig, new CommitOptions
{
PrettifyMessage = true,
}),
cancellationToken,
DefaultIOManager.BlockingTaskCreationOptions,
TaskScheduler.Current)
.ConfigureAwait(false);
}
await eventConsumer.HandleEvent(
EventType.RepoAddTestMerge,
new List <string>
{
testMergeParameters.Number.ToString(CultureInfo.InvariantCulture),
testMergeParameters.TargetCommitSha,
testMergeParameters.Comment,
},
cancellationToken)
.ConfigureAwait(false);
return(result.Status != MergeStatus.NonFastForward);
}
/// <summary>
/// Executes and populate a given <paramref name="job"/>
/// </summary>
/// <param name="job">The <see cref="CompileJob"/> to run and populate</param>
/// <param name="dreamMakerSettings">The <see cref="Api.Models.Internal.DreamMakerSettings"/> to use</param>
/// <param name="byondLock">The <see cref="IByondExecutableLock"/> to use</param>
/// <param name="repository">The <see cref="IRepository"/> to use</param>
/// <param name="remoteDeploymentManager">The <see cref="IRemoteDeploymentManager"/> to use.</param>
/// <param name="apiValidateTimeout">The timeout for validating the DMAPI</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task RunCompileJob(
Models.CompileJob job,
Api.Models.Internal.DreamMakerSettings dreamMakerSettings,
IByondExecutableLock byondLock,
IRepository repository,
IRemoteDeploymentManager remoteDeploymentManager,
uint apiValidateTimeout,
CancellationToken cancellationToken)
{
var outputDirectory = job.DirectoryName.ToString();
logger.LogTrace("Compile output GUID: {0}", outputDirectory);
try
{
// copy the repository
logger.LogTrace("Copying repository to game directory...");
var resolvedOutputDirectory = ioManager.ResolvePath(outputDirectory);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(resolvedOutputDirectory, cancellationToken).ConfigureAwait(false);
// repository closed now
// run precompile scripts
await eventConsumer.HandleEvent(
EventType.CompileStart,
new List <string>
{
resolvedOutputDirectory,
repoOrigin.ToString()
},
cancellationToken)
.ConfigureAwait(false);
// determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var foundPaths = await ioManager.GetFilesWithExtension(resolvedOutputDirectory, DmeExtension, true, cancellationToken).ConfigureAwait(false);
var foundPath = foundPaths.FirstOrDefault();
if (foundPath == default)
{
throw new JobException(ErrorCode.DreamMakerNoDme);
}
job.DmeName = foundPath.Substring(
resolvedOutputDirectory.Length + 1,
foundPath.Length - resolvedOutputDirectory.Length - DmeExtension.Length - 2); // +1 for . in extension
}
else
{
var targetDme = ioManager.ConcatPath(outputDirectory, String.Join('.', job.DmeName, DmeExtension));
var targetDmeExists = await ioManager.FileExists(targetDme, cancellationToken).ConfigureAwait(false);
if (!targetDmeExists)
{
throw new JobException(ErrorCode.DreamMakerMissingDme);
}
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
// run compiler
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
// verify api
try
{
if (exitCode != 0)
{
throw new JobException(
ErrorCode.DreamMakerExitCode,
new JobException($"Exit code: {exitCode}{Environment.NewLine}{Environment.NewLine}{job.Output}"));
}
await VerifyApi(
apiValidateTimeout,
dreamMakerSettings.ApiValidationSecurityLevel.Value,
job,
byondLock,
dreamMakerSettings.ApiValidationPort.Value,
dreamMakerSettings.RequireDMApiValidation.Value,
cancellationToken)
.ConfigureAwait(false);
}
catch (JobException)
{
// DD never validated or compile failed
await eventConsumer.HandleEvent(
EventType.CompileFailure,
new List <string>
{
resolvedOutputDirectory,
exitCode == 0 ? "1" : "0"
},
cancellationToken)
.ConfigureAwait(false);
throw;
}
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
resolvedOutputDirectory
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
// symlink in the static data
await configuration.SymlinkStaticFilesTo(resolvedOutputDirectory, cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
}
catch (Exception ex)
{
await CleanupFailedCompile(job, remoteDeploymentManager, ex).ConfigureAwait(false);
throw;
}
}
/// <summary>
/// Installs a BYOND <paramref name="version"/> if it isn't already
/// </summary>
/// <param name="version">The BYOND <see cref="Version"/> to install</param>
/// <param name="customVersionStream">Custom zip file <see cref="Stream"/> to use. Will cause a <see cref="Version.Build"/> number to be added.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task <string> InstallVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken)
{
var ourTcs = new TaskCompletionSource <object>();
Task inProgressTask;
string versionKey;
bool installed;
lock (installedVersions)
{
if (customVersionStream != null)
{
int customInstallationNumber = 1;
do
{
versionKey = $"{VersionKey(version, false)}.{customInstallationNumber++}";
}while (installedVersions.ContainsKey(versionKey));
}
else
{
versionKey = VersionKey(version, true);
}
installed = installedVersions.TryGetValue(versionKey, out inProgressTask);
if (!installed)
{
installedVersions.Add(versionKey, ourTcs.Task);
}
}
if (installed)
{
using (cancellationToken.Register(() => ourTcs.SetCanceled()))
{
await Task.WhenAny(ourTcs.Task, inProgressTask).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
return(versionKey);
}
}
if (customVersionStream != null)
{
logger.LogInformation("Installing custom BYOND version as {0}...", versionKey);
}
else if (version.Build > 0)
{
throw new JobException(ErrorCode.ByondNonExistentCustomVersion);
}
else
{
logger.LogDebug("Requested BYOND version {0} not currently installed. Doing so now...");
}
// okay up to us to install it then
try
{
await eventConsumer.HandleEvent(EventType.ByondInstallStart, new List <string> {
versionKey
}, cancellationToken).ConfigureAwait(false);
var extractPath = ioManager.ResolvePath(versionKey);
async Task DirectoryCleanup()
{
await ioManager.DeleteDirectory(extractPath, cancellationToken).ConfigureAwait(false);
await ioManager.CreateDirectory(extractPath, cancellationToken).ConfigureAwait(false);
}
var directoryCleanupTask = DirectoryCleanup();
try
{
Stream versionZipStream;
Stream downloadedStream = null;
if (customVersionStream == null)
{
var bytes = await byondInstaller.DownloadVersion(version, cancellationToken).ConfigureAwait(false);
downloadedStream = new MemoryStream(bytes);
versionZipStream = downloadedStream;
}
else
{
versionZipStream = customVersionStream;
}
using (downloadedStream)
{
await directoryCleanupTask.ConfigureAwait(false);
logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath);
await ioManager.ZipToDirectory(extractPath, versionZipStream, cancellationToken).ConfigureAwait(false);
}
await byondInstaller.InstallByond(extractPath, version, cancellationToken).ConfigureAwait(false);
// make sure to do this last because this is what tells us we have a valid version in the future
await ioManager.WriteAllBytes(
ioManager.ConcatPath(versionKey, VersionFileName),
Encoding.UTF8.GetBytes(versionKey),
cancellationToken)
.ConfigureAwait(false);
}
catch (WebException e)
{
// since the user can easily provide non-exitent version numbers, we'll turn this into a JobException
throw new JobException(ErrorCode.ByondDownloadFail, e);
}
catch (OperationCanceledException)
{
throw;
}
catch
{
await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false);
throw;
}
ourTcs.SetResult(null);
}
catch (Exception e)
{
if (!(e is OperationCanceledException))
{
await eventConsumer.HandleEvent(EventType.ByondInstallFail, new List <string> {
e.Message
}, cancellationToken).ConfigureAwait(false);
}
lock (installedVersions)
installedVersions.Remove(versionKey);
ourTcs.SetException(e);
throw;
}
return(versionKey);
}
/// <inheritdoc />
public async Task <bool?> AddTestMerge(TestMergeParameters testMergeParameters, string committerName, string committerEmail, string username, string password, Action <int> progressReporter, CancellationToken cancellationToken)
{
if (testMergeParameters == null)
{
throw new ArgumentNullException(nameof(testMergeParameters));
}
if (committerName == null)
{
throw new ArgumentNullException(nameof(committerName));
}
if (committerEmail == null)
{
throw new ArgumentNullException(nameof(committerEmail));
}
if (!IsGitHubRepository)
{
throw new InvalidOperationException("Test merging is only available on GitHub hosted origin repositories!");
}
var commitMessage = String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", testMergeParameters.Number.Value, testMergeParameters.Comment != null ? Environment.NewLine : String.Empty, testMergeParameters.Comment ?? String.Empty);
var prBranchName = String.Format(CultureInfo.InvariantCulture, "pr-{0}", testMergeParameters.Number);
var localBranchName = String.Format(CultureInfo.InvariantCulture, "pull/{0}/headrefs/heads/{1}", testMergeParameters.Number, prBranchName);
var Refspec = new List <string> {
String.Format(CultureInfo.InvariantCulture, "pull/{0}/head:{1}", testMergeParameters.Number, prBranchName)
};
var logMessage = String.Format(CultureInfo.InvariantCulture, "Merge remote pull request #{0}", testMergeParameters.Number);
var originalCommit = repository.Head;
MergeResult result = null;
var sig = new Signature(new Identity(committerName, committerEmail), DateTimeOffset.Now);
await Task.Factory.StartNew(() =>
{
try
{
try
{
var remote = repository.Network.Remotes.First();
Commands.Fetch((LibGit2Sharp.Repository)repository, remote.Name, Refspec, new FetchOptions
{
Prune = true,
OnProgress = (a) => !cancellationToken.IsCancellationRequested,
OnTransferProgress = (a) =>
{
var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2));
progressReporter?.Invoke((int)percentage);
return(!cancellationToken.IsCancellationRequested);
},
OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
CredentialsProvider = (a, b, c) => username != null ? (Credentials) new UsernamePasswordCredentials
{
Username = username,
Password = password
} : new DefaultCredentials()
}, logMessage);
}
catch (UserCancelledException) { }
cancellationToken.ThrowIfCancellationRequested();
testMergeParameters.PullRequestRevision = repository.Lookup(testMergeParameters.PullRequestRevision ?? localBranchName).Sha;
cancellationToken.ThrowIfCancellationRequested();
result = repository.Merge(testMergeParameters.PullRequestRevision, sig, new MergeOptions
{
CommitOnSuccess = commitMessage == null,
FailOnConflict = true,
FastForwardStrategy = FastForwardStrategy.NoFastForward,
SkipReuc = true
});
}
finally
{
repository.Branches.Remove(localBranchName);
}
cancellationToken.ThrowIfCancellationRequested();
if (result.Status == MergeStatus.Conflicts)
{
RawCheckout(originalCommit.CanonicalName ?? originalCommit.Tip.Sha);
cancellationToken.ThrowIfCancellationRequested();
}
repository.RemoveUntrackedFiles();
}, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false);
if (result.Status == MergeStatus.Conflicts)
{
await eventConsumer.HandleEvent(EventType.RepoMergeConflict, new List <string> {
originalCommit.Tip.Sha, testMergeParameters.PullRequestRevision, originalCommit.FriendlyName ?? UnknownReference, prBranchName
}, cancellationToken).ConfigureAwait(false);
return(false);
}
if (commitMessage != null)
{
repository.Commit(commitMessage, sig, sig, new CommitOptions
{
PrettifyMessage = true
});
}
return(true);
}