/// <inheritdoc />
public async Task StartAsync(CancellationToken cancellationToken)
{
CompileJob cj = null;
await databaseContextFactory.UseContext(async (db) =>
{
cj = await db
.CompileJobs
.AsQueryable()
.Where(x => x.Job.Instance.Id == metadata.Id)
.OrderByDescending(x => x.Job.StoppedAt)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
})
.ConfigureAwait(false);
if (cj == default(CompileJob))
{
return;
}
await LoadCompileJob(cj, cancellationToken).ConfigureAwait(false);
started = true;
// we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
}
/// <inheritdoc />
public async Task MoveInstance(Models.Instance instance, string newPath, CancellationToken cancellationToken)
{
if (newPath == null)
{
throw new ArgumentNullException(nameof(newPath));
}
if (instance.Online.Value)
{
throw new InvalidOperationException("Cannot move an online instance!");
}
var oldPath = instance.Path;
await ioManager.CopyDirectory(oldPath, newPath, null, cancellationToken).ConfigureAwait(false);
await databaseContextFactory.UseContext(db =>
{
var targetInstance = new Models.Instance
{
Id = instance.Id
};
db.Instances.Attach(targetInstance);
targetInstance.Path = newPath;
return(db.Save(cancellationToken));
}).ConfigureAwait(false);
await ioManager.DeleteDirectory(oldPath, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
public async Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken)
{
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
CompileJob finalCompileJob = null;
//now load the entire compile job tree
await databaseContextFactory.UseContext(async db => finalCompileJob = await db.CompileJobs.Where(x => x.Id == job.Id)
.Include(x => x.Job).ThenInclude(x => x.StartedBy)
.Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge).ThenInclude(x => x.MergedBy)
.Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy)
.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); //can't wait to see that query
if (finalCompileJob == null)
{
//lol git f****d
return;
}
var newProvider = await FromCompileJob(finalCompileJob, cancellationToken).ConfigureAwait(false);
if (newProvider == null)
{
return;
}
lock (this)
{
nextDmbProvider?.Dispose();
nextDmbProvider = newProvider;
newerDmbTcs.SetResult(nextDmbProvider);
newerDmbTcs = new TaskCompletionSource <object>();
}
}
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken) => databaseContextFactory.UseContext(async(db) =>
{
//where complete clause not necessary, only successful COMPILEjobs get in the db
var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id)
.OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (cj == default(CompileJob))
{
return;
}
await LoadCompileJob(cj, cancellationToken).ConfigureAwait(false);
//we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
});
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken) => databaseContextFactory.UseContext(async databaseContext =>
{
try
{
await databaseContext.Initialize(cancellationToken).ConfigureAwait(false);
await jobManager.StartAsync(cancellationToken).ConfigureAwait(false);
var dbInstances = databaseContext.Instances.Where(x => x.Online.Value)
.Include(x => x.RepositorySettings)
.Include(x => x.ChatSettings)
.ThenInclude(x => x.Channels)
.Include(x => x.DreamDaemonSettings)
.ToAsyncEnumerable();
var tasks = new List <Task>();
await dbInstances.ForEachAsync(metadata => tasks.Add(metadata.Online.Value ? OnlineInstance(metadata, cancellationToken) : Task.CompletedTask), cancellationToken).ConfigureAwait(false);
await Task.WhenAll(tasks).ConfigureAwait(false);
logger.LogInformation("Instance manager ready!");
application.Ready(null);
}
catch (OperationCanceledException)
{
logger.LogInformation("Cancelled instance manager initialization!");
}
catch (Exception e)
{
logger.LogCritical("Instance manager startup error! Exception: {0}", e);
application.Ready(e);
}
});
/// <inheritdoc />
public async Task StartAsync(CancellationToken cancellationToken)
{
CompileJob cj = null;
await databaseContextFactory.UseContext(async (db) =>
{
cj = await db
.MostRecentCompletedCompileJobOrDefault(instance, cancellationToken)
.ConfigureAwait(false);
})
.ConfigureAwait(false);
if (cj == default(CompileJob))
{
return;
}
await LoadCompileJob(cj, cancellationToken).ConfigureAwait(false);
// we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
}
/// <inheritdoc />
public async Task StartAsync(CancellationToken cancellationToken)
{
await Task.WhenAll(SetAutoUpdateInterval(metadata.AutoUpdateInterval), Configuration.StartAsync(cancellationToken), ByondManager.StartAsync(cancellationToken), Chat.StartAsync(cancellationToken), CompileJobConsumer.StartAsync(cancellationToken)).ConfigureAwait(false);
//dependent on so many things, its just safer this way
await Watchdog.StartAsync(cancellationToken).ConfigureAwait(false);
CompileJob latestCompileJob = null;
await databaseContextFactory.UseContext(async db =>
{
latestCompileJob = await db.CompileJobs.Where(x => x.Job.Instance.Id == metadata.Id && x.Job.ExceptionDetails == null).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
}).ConfigureAwait(false);
await dmbFactory.CleanUnusedCompileJobs(latestCompileJob, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
// TODO: Decomplexify
#pragma warning disable CA1506
public async Task <string> Invoke(string arguments, ChatUser user, CancellationToken cancellationToken)
{
IEnumerable <Models.TestMerge> results = null;
if (arguments.Split(' ').Any(x => x.ToUpperInvariant() == "--REPO"))
{
string head;
using (var repo = await repositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false))
{
if (repo == null)
{
return("Repository unavailable!");
}
head = repo.Head;
}
await databaseContextFactory.UseContext(
async db => results = await db
.RevisionInformations
.AsQueryable()
.Where(x => x.Instance.Id == instance.Id && x.CommitSha == head)
.SelectMany(x => x.ActiveTestMerges)
.Select(x => x.TestMerge)
.Select(x => new Models.TestMerge
{
Number = x.Number,
PullRequestRevision = x.PullRequestRevision
})
.ToListAsync(cancellationToken)
.ConfigureAwait(false))
.ConfigureAwait(false);
}
else
{
if (!watchdog.Running)
{
return("Server offline!");
}
results = watchdog.ActiveCompileJob?.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).ToList() ?? new List <Models.TestMerge>();
}
return(!results.Any() ? "None!" : String.Join(", ", results.Select(x => String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7)))));
}
/// <inheritdoc />
public async Task StartAsync(CancellationToken cancellationToken)
{
var reattachInfo = await reattachInfoHandler.Load(cancellationToken).ConfigureAwait(false);
if (!autoStart && reattachInfo == null)
{
return;
}
long?adminUserId = null;
await databaseContextFactory.UseContext(async db => adminUserId = await db.Users
.Where(x => x.CanonicalName == Api.Models.User.AdminName.ToUpperInvariant())
.Select(x => x.Id)
.FirstAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false);
var job = new Models.Job
{
StartedBy = new Models.User
{
Id = adminUserId.Value
},
Instance = new Models.Instance
{
Id = instance.Id
},
Description = "Instance startup watchdog launch",
CancelRight = (ulong)DreamDaemonRights.Shutdown,
CancelRightsType = RightsType.DreamDaemon
};
await jobManager.RegisterOperation(job, async (j, databaseContext, progressFunction, ct) =>
{
using (await SemaphoreSlimContext.Lock(Semaphore, ct).ConfigureAwait(false))
await LaunchImplNoLock(true, true, reattachInfo, ct).ConfigureAwait(false);
}, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Pull the repository and compile for every set of given <paramref name="minutes"/>
/// </summary>
/// <param name="minutes">How many minutes the operation should repeat. Does not include running time</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
#pragma warning disable CA1502 // TODO: Decomplexify
async Task TimerLoop(uint minutes, CancellationToken cancellationToken)
{
while (true)
{
try
{
await Task.Delay(TimeSpan.FromMinutes(minutes > Int32.MaxValue ? Int32.MaxValue : (int)minutes), cancellationToken).ConfigureAwait(false);
logger.LogInformation("Beginning auto update...");
await eventConsumer.HandleEvent(EventType.InstanceAutoUpdateStart, new List <string>(), cancellationToken).ConfigureAwait(false);
try
{
Models.User user = null;
await databaseContextFactory.UseContext(async (db) => user = await db.Users.Where(x => x.CanonicalName == Api.Models.User.AdminName.ToUpperInvariant()).FirstAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false);
var repositoryUpdateJob = new Job
{
Instance = new Models.Instance
{
Id = metadata.Id
},
Description = "Scheduled repository update",
CancelRightsType = RightsType.Repository,
CancelRight = (ulong)RepositoryRights.CancelPendingChanges,
StartedBy = user
};
string deploySha = null;
await jobManager.RegisterOperation(repositoryUpdateJob, async (paramJob, databaseContext, progressReporter, jobCancellationToken) =>
{
var repositorySettingsTask = databaseContext.RepositorySettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(jobCancellationToken);
// assume 5 steps with synchronize
const int ProgressSections = 7;
const int ProgressStep = 100 / ProgressSections;
const int NumSteps = 3;
var doneSteps = 0;
Action <int> NextProgressReporter()
{
var tmpDoneSteps = doneSteps;
++doneSteps;
return(progress => progressReporter((progress + (100 * tmpDoneSteps)) / NumSteps));
}
using (var repo = await RepositoryManager.LoadRepository(jobCancellationToken).ConfigureAwait(false))
{
if (repo == null)
{
logger.LogTrace("Aborting repo update, no repository!");
return;
}
var startSha = repo.Head;
if (!repo.Tracking)
{
logger.LogTrace("Aborting repo update, active ref not tracking any remote branch!");
deploySha = startSha;
return;
}
var repositorySettings = await repositorySettingsTask.ConfigureAwait(false);
// the main point of auto update is to pull the remote
await repo.FetchOrigin(repositorySettings.AccessUser, repositorySettings.AccessToken, NextProgressReporter(), jobCancellationToken).ConfigureAwait(false);
RevisionInformation currentRevInfo = null;
bool hasDbChanges = false;
Task <RevisionInformation> LoadRevInfo() => databaseContext.RevisionInformations
.Where(x => x.CommitSha == startSha && x.Instance.Id == metadata.Id)
.Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge)
.FirstOrDefaultAsync(cancellationToken);
async Task UpdateRevInfo(string currentHead, bool onOrigin)
{
if (currentRevInfo == null)
{
currentRevInfo = await LoadRevInfo().ConfigureAwait(false);
}
if (currentRevInfo == default)
{
logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, currentHead);
onOrigin = true;
}
var attachedInstance = new Models.Instance
{
Id = metadata.Id
};
var oldRevInfo = currentRevInfo;
currentRevInfo = new RevisionInformation
{
CommitSha = currentHead,
OriginCommitSha = onOrigin ? currentHead : oldRevInfo.OriginCommitSha,
Instance = attachedInstance
};
if (!onOrigin)
{
currentRevInfo.ActiveTestMerges = new List <RevInfoTestMerge>(oldRevInfo.ActiveTestMerges);
}
databaseContext.Instances.Attach(attachedInstance);
databaseContext.RevisionInformations.Add(currentRevInfo);
hasDbChanges = true;
}
// take appropriate auto update actions
bool shouldSyncTracked;
if (repositorySettings.AutoUpdatesKeepTestMerges.Value)
{
logger.LogTrace("Preserving test merges...");
var currentRevInfoTask = LoadRevInfo();
var result = await repo.MergeOrigin(repositorySettings.CommitterName, repositorySettings.CommitterEmail, NextProgressReporter(), jobCancellationToken).ConfigureAwait(false);
if (!result.HasValue)
{
throw new JobException("Merge conflict while preserving test merges!");
}
currentRevInfo = await currentRevInfoTask.ConfigureAwait(false);
var lastRevInfoWasOriginCommit = currentRevInfo == default || currentRevInfo.CommitSha == currentRevInfo.OriginCommitSha;
var stillOnOrigin = result.Value && lastRevInfoWasOriginCommit;
var currentHead = repo.Head;
if (currentHead != startSha)
{
await UpdateRevInfo(currentHead, stillOnOrigin).ConfigureAwait(false);
shouldSyncTracked = stillOnOrigin;
}
else
{
shouldSyncTracked = false;
}
}
else
{
logger.LogTrace("Not preserving test merges...");
await repo.ResetToOrigin(NextProgressReporter(), jobCancellationToken).ConfigureAwait(false);
var currentHead = repo.Head;
currentRevInfo = await databaseContext.RevisionInformations
.Where(x => x.CommitSha == currentHead && x.Instance.Id == metadata.Id)
.FirstOrDefaultAsync(jobCancellationToken).ConfigureAwait(false);
if (currentHead != startSha && currentRevInfo != default)
{
await UpdateRevInfo(currentHead, true).ConfigureAwait(false);
}
shouldSyncTracked = true;
}
// synch if necessary
if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head)
{
var pushedOrigin = await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, repositorySettings.CommitterName, repositorySettings.CommitterEmail, NextProgressReporter(), shouldSyncTracked, jobCancellationToken).ConfigureAwait(false);
var currentHead = repo.Head;
if (currentHead != currentRevInfo.CommitSha)
{
await UpdateRevInfo(currentHead, pushedOrigin).ConfigureAwait(false);
}
}
if (hasDbChanges)
{
try
{
await databaseContext.Save(cancellationToken).ConfigureAwait(false);
}
catch
{
await repo.ResetToSha(startSha, progressReporter, default).ConfigureAwait(false);
throw;
}
}
progressReporter(5 * ProgressStep);
deploySha = repo.Head;
}
}, cancellationToken).ConfigureAwait(false);
await jobManager.WaitForJobCompletion(repositoryUpdateJob, user, cancellationToken, default).ConfigureAwait(false);
if (deploySha == null)
{
logger.LogTrace("Aborting auto update, repository error!");
continue;
}
if (deploySha == LatestCompileJob()?.RevisionInformation.CommitSha)
{
logger.LogTrace("Aborting auto update, same revision as latest CompileJob");
continue;
}
// finally set up the job
var compileProcessJob = new Job
{
StartedBy = user,
Instance = repositoryUpdateJob.Instance,
Description = "Scheduled code deployment",
CancelRightsType = RightsType.DreamMaker,
CancelRight = (ulong)DreamMakerRights.CancelCompile
};
await jobManager.RegisterOperation(compileProcessJob, CompileProcess, cancellationToken).ConfigureAwait(false);
await jobManager.WaitForJobCompletion(compileProcessJob, user, cancellationToken, default).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
logger.LogDebug("Cancelled auto update job!");
throw;
}
catch (Exception e)
{
logger.LogWarning("Error in auto update loop! Exception: {0}", e);
continue;
}
}
catch (OperationCanceledException)
{
break;
}
}
logger.LogTrace("Leaving auto update loop...");
}
/// <inheritdoc />
public async Task StartDeployment(IRepository repository, CompileJob compileJob, CancellationToken cancellationToken)
{
if (repository == null)
{
throw new ArgumentNullException(nameof(repository));
}
if (compileJob == null)
{
throw new ArgumentNullException(nameof(compileJob));
}
if (!repository.IsGitHubRepository)
{
logger.LogTrace("Not managing deployment as this is not a GitHub repo");
return;
}
logger.LogTrace("Starting deployment...");
RepositorySettings repositorySettings = null;
await databaseContextFactory.UseContext(
async databaseContext =>
repositorySettings = await databaseContext
.RepositorySettings
.AsQueryable()
.Where(x => x.InstanceId == metadata.Id)
.FirstAsync(cancellationToken)
.ConfigureAwait(false))
.ConfigureAwait(false);
var gitHubClient = repositorySettings.AccessToken == null
? gitHubClientFactory.CreateClient()
: gitHubClientFactory.CreateClient(repositorySettings.AccessToken);
var repositoryTask = gitHubClient
.Repository
.Get(
repository.GitHubOwner,
repository.GitHubRepoName);
if (repositorySettings.CreateGitHubDeployments.Value)
{
logger.LogTrace("Creating deployment...");
var deployment = await gitHubClient
.Repository
.Deployment
.Create(
repository.GitHubOwner,
repository.GitHubRepoName,
new NewDeployment(compileJob.RevisionInformation.CommitSha)
{
AutoMerge = false,
Description = "TGS Game Deployment",
Environment = $"TGS: {metadata.Name}",
ProductionEnvironment = true,
RequiredContexts = new Collection <string>()
})
.WithToken(cancellationToken)
.ConfigureAwait(false);
compileJob.GitHubDeploymentId = deployment.Id;
logger.LogDebug("Created deployment ID {0}", deployment.Id);
await gitHubClient
.Repository
.Deployment
.Status
.Create(
repository.GitHubOwner,
repository.GitHubRepoName,
deployment.Id,
new NewDeploymentStatus(DeploymentState.InProgress)
{
Description = "The project is being deployed",
AutoInactive = false
})
.WithToken(cancellationToken)
.ConfigureAwait(false);
logger.LogTrace("In-progress deployment status created");
}
else
{
logger.LogTrace("Not creating deployment");
}
try
{
var gitHubRepo = await repositoryTask
.WithToken(cancellationToken)
.ConfigureAwait(false);
compileJob.GitHubRepoId = gitHubRepo.Id;
logger.LogTrace("Set GitHub ID as {0}", compileJob.GitHubRepoId);
}
catch (RateLimitExceededException ex) when(!repositorySettings.CreateGitHubDeployments.Value)
{
logger.LogWarning(ex, "Unable to set compile job repository ID!");
}
}
/// <inheritdoc />
public Task Save(ReattachInformation reattachInformation, CancellationToken cancellationToken) => databaseContextFactory.UseContext(async(db) =>
{
if (reattachInformation == null)
{
throw new ArgumentNullException(nameof(reattachInformation));
}
logger.LogDebug("Saving reattach information: {0}...", reattachInformation);
await db
.ReattachInformations
.AsQueryable()
.Where(x => x.CompileJob.Job.Instance.Id == metadata.Id)
.DeleteAsync(cancellationToken)
.ConfigureAwait(false);
var dbReattachInfo = new Models.ReattachInformation
{
AccessIdentifier = reattachInformation.AccessIdentifier,
CompileJobId = reattachInformation.Dmb.CompileJob.Id,
Port = reattachInformation.Port,
ProcessId = reattachInformation.ProcessId,
RebootState = reattachInformation.RebootState,
LaunchSecurityLevel = reattachInformation.LaunchSecurityLevel
};
db.ReattachInformations.Add(dbReattachInfo);
await db.Save(cancellationToken).ConfigureAwait(false);
});
