public void Dispose() => cleanupCts.Dispose(); // we don't dispose nextDmbProvider here, since it might be the only thing we have /// <summary> /// Delete the <see cref="Api.Models.Internal.CompileJob.DirectoryName"/> of <paramref name="job"/> /// </summary> /// <param name="job">The <see cref="CompileJob"/> to clean</param> void CleanJob(CompileJob job) { async Task HandleCleanup() { var deleteJob = ioManager.DeleteDirectory(job.DirectoryName.ToString(), cleanupCts.Token); Task otherTask; // lock (this) //already locked below otherTask = cleanupTask; await Task.WhenAll(otherTask, deleteJob).ConfigureAwait(false); } lock (this) if (!jobLockCounts.TryGetValue(job.Id, out var currentVal) || currentVal == 1) { jobLockCounts.Remove(job.Id); logger.LogDebug("Cleaning compile job {0} => {1}", job.Id, job.DirectoryName); cleanupTask = HandleCleanup(); } else { var decremented = --jobLockCounts[job.Id]; logger.LogTrace("Compile job {0} lock count now: {1}", job.Id, decremented); } }
public void Dispose() => cleanupCts.Dispose(); // we don't dispose nextDmbProvider here, since it might be the only thing we have /// <summary> /// Delete the <see cref="Api.Models.Internal.CompileJob.DirectoryName"/> of <paramref name="job"/> /// </summary> /// <param name="job">The <see cref="CompileJob"/> to clean</param> void CleanJob(CompileJob job) { async Task HandleCleanup() { var deleteJob = ioManager.DeleteDirectory(job.DirectoryName.ToString(), cleanupCts.Token); var remoteDeploymentManager = remoteDeploymentManagerFactory.CreateRemoteDeploymentManager( metadata, job); // DCT: None available var deploymentJob = remoteDeploymentManager.MarkInactive(job, default); var otherTask = cleanupTask; await Task.WhenAll(otherTask, deleteJob, deploymentJob).ConfigureAwait(false); } lock (jobLockCounts) if (!jobLockCounts.TryGetValue(job.Id, out var currentVal) || currentVal == 1) { jobLockCounts.Remove(job.Id); logger.LogDebug("Cleaning lock-free compile job {0} => {1}", job.Id, job.DirectoryName); cleanupTask = HandleCleanup(); } else { var decremented = --jobLockCounts[job.Id]; logger.LogTrace("Compile job {0} lock count now: {1}", job.Id, decremented); } }
/// <inheritdoc /> public async Task MoveInstance(Models.Instance instance, string newPath, CancellationToken cancellationToken) { if (newPath == null) { throw new ArgumentNullException(nameof(newPath)); } if (instance.Online.Value) { throw new InvalidOperationException("Cannot move an online instance!"); } var oldPath = instance.Path; await ioManager.CopyDirectory(oldPath, newPath, null, cancellationToken).ConfigureAwait(false); await databaseContextFactory.UseContext(db => { var targetInstance = new Models.Instance { Id = instance.Id }; db.Instances.Attach(targetInstance); targetInstance.Path = newPath; return(db.Save(cancellationToken)); }).ConfigureAwait(false); await ioManager.DeleteDirectory(oldPath, cancellationToken).ConfigureAwait(false); }
/// <inheritdoc /> public async Task <bool> ApplyUpdate(byte[] updateZipData, IIOManager ioManager, CancellationToken cancellationToken) { if (updatePath == null) { return(false); } using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { if (updated) { throw new InvalidOperationException("ApplyUpdate has already been called!"); } updated = true; try { await ioManager.ZipToDirectory(updatePath, updateZipData, cancellationToken).ConfigureAwait(false); } catch { try { //important to not leave this directory around if possible await ioManager.DeleteDirectory(updatePath, default).ConfigureAwait(false); } catch { } updated = false; throw; } Restart(); return(true); } }
/// <summary> /// Cleans up a failed compile <paramref name="job"/>. /// </summary> /// <param name="job">The running <see cref="CompileJob"/>.</param> /// <param name="remoteDeploymentManager">The <see cref="IRemoteDeploymentManager"/> associated with the <paramref name="job"/>.</param> /// <param name="exception">The <see cref="Exception"/> that was thrown.</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task CleanupFailedCompile(Models.CompileJob job, IRemoteDeploymentManager remoteDeploymentManager, Exception exception) { async Task CleanDir() { logger.LogTrace("Cleaning compile directory..."); var jobPath = job.DirectoryName.ToString(); try { // DCT: None available await ioManager.DeleteDirectory(jobPath, default).ConfigureAwait(false); } catch (Exception e) { logger.LogWarning(e, "Error cleaning up compile directory {0}!", ioManager.ResolvePath(jobPath)); } } // DCT: None available await Task.WhenAll( CleanDir(), remoteDeploymentManager.FailDeployment( job, FormatExceptionForUsers(exception), default)) .ConfigureAwait(false); }
/// <summary> /// Make the <see cref="WindowsSwappableDmbProvider"/> active by replacing the live link with our <see cref="CompileJob"/>. /// </summary> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param> /// <returns>A <see cref="Task"/> representing the running operation.</returns> public async Task MakeActive(CancellationToken cancellationToken) { // Note this comment from TGS3: // These next two lines should be atomic but this is the best we can do await ioManager.DeleteDirectory(LiveGameDirectory, cancellationToken).ConfigureAwait(false); await symlinkFactory.CreateSymbolicLink(baseProvider.PrimaryDirectory, ioManager.ResolvePath(LiveGameDirectory), cancellationToken).ConfigureAwait(false); }
/// <inheritdoc /> public async Task CleanCache(CancellationToken cancellationToken) { try { await ioManager.DeleteDirectory(ByondCachePath, cancellationToken).ConfigureAwait(false); } catch (Exception e) { logger.LogWarning("Error deleting BYOND cache! Exception: {0}", e); } }
/// <inheritdoc /> public async Task CleanCache(CancellationToken cancellationToken) { try { await ioManager.DeleteDirectory(ioManager.ConcatPath(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "byond/cache"), cancellationToken).ConfigureAwait(false); } catch (Exception e) { logger.LogWarning("Error deleting BYOND cache! Exception: {0}", e); } }
async Task <string> ValidateNonExistantSqliteDBName(string databaseName, CancellationToken cancellationToken) { var resolvedPath = ioManager.ResolvePath(databaseName); try { var directoryName = ioManager.GetDirectoryName(resolvedPath); bool directoryExisted = await ioManager.DirectoryExists(directoryName, cancellationToken).ConfigureAwait(false); await ioManager.CreateDirectory(directoryName, cancellationToken).ConfigureAwait(false); try { await ioManager.WriteAllBytes(resolvedPath, Array.Empty <byte>(), cancellationToken).ConfigureAwait(false); } catch { if (!directoryExisted) { await ioManager.DeleteDirectory(directoryName, cancellationToken).ConfigureAwait(false); } throw; } } catch (IOException) { return(null); } if (!Path.IsPathRooted(databaseName)) { await console.WriteAsync("Note, this relative path (currently) resolves to the following:", true, cancellationToken).ConfigureAwait(false); await console.WriteAsync(resolvedPath, true, cancellationToken).ConfigureAwait(false); bool writeResolved = await PromptYesNo( "Would you like to save the relative path in the configuration? If not, the full path will be saved. (y/n): ", cancellationToken) .ConfigureAwait(false); if (writeResolved) { databaseName = resolvedPath; } } await ioManager.DeleteFile(databaseName, cancellationToken).ConfigureAwait(false); return(databaseName); }
/// <summary> /// Cleans up a failed compile <paramref name="job"/> /// </summary> /// <param name="job">The running <see cref="CompileJob"/></param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task CleanupFailedCompile(Models.CompileJob job) { logger.LogTrace("Cleaning compile directory..."); var jobPath = job.DirectoryName.ToString(); try { await ioManager.DeleteDirectory(jobPath, CancellationToken.None).ConfigureAwait(false); } catch (Exception e) { logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(jobPath), e); } }
/// <inheritdoc /> public async Task <bool> ApplyUpdate(Version version, byte[] updateZipData, IIOManager ioManager, CancellationToken cancellationToken) { if (version == null) { throw new ArgumentNullException(nameof(version)); } if (updateZipData == null) { throw new ArgumentNullException(nameof(updateZipData)); } if (ioManager == null) { throw new ArgumentNullException(nameof(ioManager)); } if (updatePath == null) { return(false); } using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { if (updated) { throw new InvalidOperationException("ApplyUpdate has already been called!"); } updated = true; try { await ioManager.ZipToDirectory(updatePath, updateZipData, cancellationToken).ConfigureAwait(false); } catch (Exception e) { updated = false; try { //important to not leave this directory around if possible await ioManager.DeleteDirectory(updatePath, default).ConfigureAwait(false); } catch (Exception e2) { throw new AggregateException(e, e2); } throw; } await Restart(version).ConfigureAwait(false); return(true); } }
/// <inheritdoc /> public async Task MoveInstance(Models.Instance instance, string newPath, CancellationToken cancellationToken) { if (newPath == null) { throw new ArgumentNullException(nameof(newPath)); } if (instance.Online.Value) { throw new InvalidOperationException("Cannot move an online instance!"); } var oldPath = instance.Path; await ioManager.CopyDirectory(oldPath, newPath, null, cancellationToken).ConfigureAwait(false); instance.Path = ioManager.ResolvePath(newPath); await ioManager.DeleteDirectory(oldPath, cancellationToken).ConfigureAwait(false); }
/// <summary> /// Cleans up a failed compile <paramref name="job"/> /// </summary> /// <param name="job">The running <see cref="Models.CompileJob"/></param> /// <param name="cancelled">If the <paramref name="job"/> was cancelled</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task CleanupFailedCompile(Models.CompileJob job, bool cancelled, CancellationToken cancellationToken) { logger.LogTrace("Cleaning compile directory..."); var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken); var jobPath = job.DirectoryName.ToString(); try { await ioManager.DeleteDirectory(jobPath, CancellationToken.None).ConfigureAwait(false); } catch (Exception e) { logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(jobPath), e); } await chatTask.ConfigureAwait(false); }
/// <inheritdoc /> public async Task SymlinkStaticFilesTo(string destination, CancellationToken cancellationToken) { async Task SymlinkBase(bool files) { Task <IReadOnlyList <string> > task; if (files) { task = ioManager.GetFiles(GameStaticFilesSubdirectory, cancellationToken); } else { task = ioManager.GetDirectories(GameStaticFilesSubdirectory, cancellationToken); } var entries = await task.ConfigureAwait(false); await Task.WhenAll(task.Result.Select(async x => { var destPath = ioManager.ConcatPath(destination, ioManager.GetFileName(x)); logger.LogTrace("Symlinking {0} to {1}...", x, destPath); var fileExistsTask = ioManager.FileExists(destPath, cancellationToken); if (await ioManager.DirectoryExists(destPath, cancellationToken).ConfigureAwait(false)) { await ioManager.DeleteDirectory(destPath, cancellationToken).ConfigureAwait(false); } var fileExists = await fileExistsTask.ConfigureAwait(false); if (fileExists) { await ioManager.DeleteFile(destPath, cancellationToken).ConfigureAwait(false); } await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(x), ioManager.ResolvePath(destPath), cancellationToken).ConfigureAwait(false); })).ConfigureAwait(false); } using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { await EnsureDirectories(cancellationToken).ConfigureAwait(false); await Task.WhenAll(SymlinkBase(true), SymlinkBase(false)).ConfigureAwait(false); } }
/// <inheritdoc /> public async Task <ILocalRepository> GetRepository(Octokit.Repository repository, Func <int, Task> onCloneProgress, Func <Task> onOperationBlocked, CancellationToken cancellationToken) { if (repository == null) { throw new ArgumentNullException(nameof(repository)); } var repoPath = ioManager.ConcatPath(repository.Owner.Login, repository.Name); TaskCompletionSource <object> usageTask = null; try { return(await TryLoadRepository(repoPath, onOperationBlocked, tcs => usageTask = tcs, cancellationToken).ConfigureAwait(false)); } catch (LibGit2SharpException e) { logger.LogWarning(e, "Failed to load repository {0}/{1}! Cloning...", repository.Owner.Login, repository.Name); } //so the repo failed to load and now we're holding our queue spot in usageTask //reclone it try { await ioManager.DeleteDirectory(repoPath, cancellationToken).ConfigureAwait(false); await ioManager.CreateDirectory(repoPath, cancellationToken).ConfigureAwait(false); await repositoryOperations.Clone(repository.CloneUrl, ioManager.ResolvePath(repoPath), onCloneProgress, cancellationToken).ConfigureAwait(false); return(await localRepositoryFactory.CreateLocalRepository(ioManager.ResolvePath(repoPath), usageTask, cancellationToken).ConfigureAwait(false)); } catch { //ok we can't do anything else, clear our queue spot usageTask.SetResult(null); throw; } }
/// <summary> /// Installs a BYOND <paramref name="version"/> if it isn't already /// </summary> /// <param name="version">The BYOND <see cref="Version"/> to install</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task InstallVersion(Version version, CancellationToken cancellationToken) { var ourTcs = new TaskCompletionSource <object>(); Task inProgressTask; var versionKey = VersionKey(version); bool installed; lock (installedVersions) { installed = installedVersions.TryGetValue(versionKey, out inProgressTask); if (!installed) { installedVersions.Add(versionKey, ourTcs.Task); } } if (installed) { using (cancellationToken.Register(() => ourTcs.SetCanceled())) { await Task.WhenAny(ourTcs.Task, inProgressTask).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); return; } } else { logger.LogDebug("Requested BYOND version {0} not currently installed. Doing so now..."); } // okay up to us to install it then try { await eventConsumer.HandleEvent(EventType.ByondInstallStart, new List <string> { versionKey }, cancellationToken).ConfigureAwait(false); var downloadTask = byondInstaller.DownloadVersion(version, cancellationToken); await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false); try { var download = await downloadTask.ConfigureAwait(false); await ioManager.CreateDirectory(versionKey, cancellationToken).ConfigureAwait(false); var extractPath = ioManager.ResolvePath(versionKey); logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath); await ioManager.ZipToDirectory(extractPath, download, cancellationToken).ConfigureAwait(false); await byondInstaller.InstallByond(extractPath, version, cancellationToken).ConfigureAwait(false); // make sure to do this last because this is what tells us we have a valid version in the future await ioManager.WriteAllBytes(ioManager.ConcatPath(versionKey, VersionFileName), Encoding.UTF8.GetBytes(version.ToString()), cancellationToken).ConfigureAwait(false); } catch (WebException e) { // since the user can easily provide non-exitent version numbers, we'll turn this into a JobException throw new JobException(ErrorCode.ByondDownloadFail, e); } catch (OperationCanceledException) { throw; } catch { await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false); throw; } ourTcs.SetResult(null); } catch (Exception e) { if (!(e is OperationCanceledException)) { await eventConsumer.HandleEvent(EventType.ByondInstallFail, new List <string> { e.Message }, cancellationToken).ConfigureAwait(false); } lock (installedVersions) installedVersions.Remove(versionKey); ourTcs.SetException(e); throw; } }
/// <inheritdoc /> public async Task <IRepository> CloneRepository(Uri url, string initialBranch, string username, string password, Action <int> progressReporter, CancellationToken cancellationToken) { lock (this) { if (CloneInProgress) { throw new InvalidOperationException("The repository is already being cloned!"); } CloneInProgress = true; } try { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) if (!await ioManager.DirectoryExists(".", cancellationToken).ConfigureAwait(false)) { try { await Task.Factory.StartNew(() => { string path = null; try { path = LibGit2Sharp.Repository.Clone(url.ToString(), ioManager.ResolvePath("."), new CloneOptions { OnProgress = (a) => !cancellationToken.IsCancellationRequested, OnTransferProgress = (a) => { var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); progressReporter((int)percentage); return(!cancellationToken.IsCancellationRequested); }, RecurseSubmodules = true, OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested, BranchName = initialBranch, CredentialsProvider = (a, b, c) => username != null ? (Credentials) new UsernamePasswordCredentials { Username = username, Password = password } : new DefaultCredentials() }); } catch (UserCancelledException) { } cancellationToken.ThrowIfCancellationRequested(); }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); } catch { try { await ioManager.DeleteDirectory(".", default).ConfigureAwait(false); } catch { } throw; } } else { return(null); } } finally { CloneInProgress = false; } return(await LoadRepository(cancellationToken).ConfigureAwait(false)); }
/// <summary> /// Installs a BYOND <paramref name="version"/> if it isn't already /// </summary> /// <param name="version">The BYOND <see cref="Version"/> to install</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task InstallVersion(Version version, CancellationToken cancellationToken) { var ourTcs = new TaskCompletionSource <object>(); Task inProgressTask; var versionKey = VersionKey(version); bool installed; lock (installedVersions) { installed = installedVersions.TryGetValue(versionKey, out inProgressTask); if (!installed) { installedVersions.Add(versionKey, ourTcs.Task); } } if (installed) { using (cancellationToken.Register(() => ourTcs.SetCanceled())) { await Task.WhenAny(ourTcs.Task, inProgressTask).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); return; } } try { var downloadTask = byondInstaller.DownloadVersion(version, cancellationToken); //okay up to us to install it then await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false); await ioManager.CreateDirectory(versionKey, cancellationToken).ConfigureAwait(false); try { //byond can just decide to corrupt the zip fnr //(or maybe our downloader is a s***e) //either way try a few times for (var I = 0; I < 3; ++I) { var download = await downloadTask.ConfigureAwait(false); try { await ioManager.ZipToDirectory(versionKey, download, cancellationToken).ConfigureAwait(false); break; } catch (OperationCanceledException) { throw; } catch { if (I == 2) { throw; } downloadTask = byondInstaller.DownloadVersion(version, cancellationToken); } } await byondInstaller.InstallByond(ioManager.ResolvePath(versionKey), version, cancellationToken).ConfigureAwait(false); //make sure to do this last because this is what tells us we have a valid version in the future await ioManager.WriteAllBytes(ioManager.ConcatPath(versionKey, VersionFileName), Encoding.UTF8.GetBytes(version.ToString()), cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { throw; } catch { await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false); throw; } ourTcs.SetResult(null); } catch (Exception e) { lock (installedVersions) installedVersions.Remove(versionKey); ourTcs.SetException(e); throw; } }
/// <summary> /// Installs a BYOND <paramref name="version"/> if it isn't already /// </summary> /// <param name="version">The BYOND <see cref="Version"/> to install</param> /// <param name="customVersionStream">Custom zip file <see cref="Stream"/> to use. Will cause a <see cref="Version.Build"/> number to be added.</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task <string> InstallVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken) { var ourTcs = new TaskCompletionSource <object>(); Task inProgressTask; string versionKey; bool installed; lock (installedVersions) { if (customVersionStream != null) { int customInstallationNumber = 1; do { versionKey = $"{VersionKey(version, false)}.{customInstallationNumber++}"; }while (installedVersions.ContainsKey(versionKey)); } else { versionKey = VersionKey(version, true); } installed = installedVersions.TryGetValue(versionKey, out inProgressTask); if (!installed) { installedVersions.Add(versionKey, ourTcs.Task); } } if (installed) { using (cancellationToken.Register(() => ourTcs.SetCanceled())) { await Task.WhenAny(ourTcs.Task, inProgressTask).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); return(versionKey); } } if (customVersionStream != null) { logger.LogInformation("Installing custom BYOND version as {0}...", versionKey); } else if (version.Build > 0) { throw new JobException(ErrorCode.ByondNonExistentCustomVersion); } else { logger.LogDebug("Requested BYOND version {0} not currently installed. Doing so now..."); } // okay up to us to install it then try { await eventConsumer.HandleEvent(EventType.ByondInstallStart, new List <string> { versionKey }, cancellationToken).ConfigureAwait(false); var extractPath = ioManager.ResolvePath(versionKey); async Task DirectoryCleanup() { await ioManager.DeleteDirectory(extractPath, cancellationToken).ConfigureAwait(false); await ioManager.CreateDirectory(extractPath, cancellationToken).ConfigureAwait(false); } var directoryCleanupTask = DirectoryCleanup(); try { Stream versionZipStream; Stream downloadedStream = null; if (customVersionStream == null) { var bytes = await byondInstaller.DownloadVersion(version, cancellationToken).ConfigureAwait(false); downloadedStream = new MemoryStream(bytes); versionZipStream = downloadedStream; } else { versionZipStream = customVersionStream; } using (downloadedStream) { await directoryCleanupTask.ConfigureAwait(false); logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath); await ioManager.ZipToDirectory(extractPath, versionZipStream, cancellationToken).ConfigureAwait(false); } await byondInstaller.InstallByond(extractPath, version, cancellationToken).ConfigureAwait(false); // make sure to do this last because this is what tells us we have a valid version in the future await ioManager.WriteAllBytes( ioManager.ConcatPath(versionKey, VersionFileName), Encoding.UTF8.GetBytes(versionKey), cancellationToken) .ConfigureAwait(false); } catch (WebException e) { // since the user can easily provide non-exitent version numbers, we'll turn this into a JobException throw new JobException(ErrorCode.ByondDownloadFail, e); } catch (OperationCanceledException) { throw; } catch { await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false); throw; } ourTcs.SetResult(null); } catch (Exception e) { if (!(e is OperationCanceledException)) { await eventConsumer.HandleEvent(EventType.ByondInstallFail, new List <string> { e.Message }, cancellationToken).ConfigureAwait(false); } lock (installedVersions) installedVersions.Remove(versionKey); ourTcs.SetException(e); throw; } return(versionKey); }
/// <inheritdoc /> public async Task SymlinkStaticFilesTo(string destination, CancellationToken cancellationToken) { async Task <IReadOnlyList <string> > GetIgnoreFiles() { var ignoreFileBytes = await ioManager.ReadAllBytes(StaticIgnorePath(), cancellationToken).ConfigureAwait(false); var ignoreFileText = Encoding.UTF8.GetString(ignoreFileBytes); var results = new List <string> { StaticIgnoreFile }; //we don't want to lose trailing whitespace on linux using (var reader = new StringReader(ignoreFileText)) { cancellationToken.ThrowIfCancellationRequested(); var line = await reader.ReadLineAsync().ConfigureAwait(false); if (!String.IsNullOrEmpty(line)) { results.Add(line); } } return(results); }; IReadOnlyList <string> ignoreFiles; async Task SymlinkBase(bool files) { Task <IReadOnlyList <string> > task; if (files) { task = ioManager.GetFiles(GameStaticFilesSubdirectory, cancellationToken); } else { task = ioManager.GetDirectories(GameStaticFilesSubdirectory, cancellationToken); } var entries = await task.ConfigureAwait(false); await Task.WhenAll(entries.Select(async x => { var fileName = ioManager.GetFileName(x); bool ignored; if (platformIdentifier.IsWindows) { //need to normalize ignored = ignoreFiles.Any(y => fileName.ToUpperInvariant() == y.ToUpperInvariant()); } else { ignored = ignoreFiles.Any(y => fileName == y); } if (ignored) { logger.LogTrace("Ignoring static file {0}...", fileName); return; } var destPath = ioManager.ConcatPath(destination, fileName); logger.LogTrace("Symlinking {0} to {1}...", x, destPath); var fileExistsTask = ioManager.FileExists(destPath, cancellationToken); if (await ioManager.DirectoryExists(destPath, cancellationToken).ConfigureAwait(false)) { await ioManager.DeleteDirectory(destPath, cancellationToken).ConfigureAwait(false); } var fileExists = await fileExistsTask.ConfigureAwait(false); if (fileExists) { await ioManager.DeleteFile(destPath, cancellationToken).ConfigureAwait(false); } await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(x), ioManager.ResolvePath(destPath), cancellationToken).ConfigureAwait(false); })).ConfigureAwait(false); } using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { await EnsureDirectories(cancellationToken).ConfigureAwait(false); ignoreFiles = await GetIgnoreFiles().ConfigureAwait(false); await Task.WhenAll(SymlinkBase(true), SymlinkBase(false)).ConfigureAwait(false); } }
/// <inheritdoc /> public async Task <Models.CompileJob> Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action <int> progressReporter, TimeSpan?estimatedDuration, CancellationToken cancellationToken) { if (revisionInformation == null) { throw new ArgumentNullException(nameof(revisionInformation)); } if (dreamMakerSettings == null) { throw new ArgumentNullException(nameof(dreamMakerSettings)); } if (repository == null) { throw new ArgumentNullException(nameof(repository)); } if (progressReporter == null) { throw new ArgumentNullException(nameof(progressReporter)); } if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe) { throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!"); } logger.LogTrace("Begin Compile"); var job = new Models.CompileJob { DirectoryName = Guid.NewGuid(), DmeName = dreamMakerSettings.ProjectName, RevisionInformation = revisionInformation }; logger.LogTrace("Compile output GUID: {0}", job.DirectoryName); lock (this) { if (compiling) { throw new JobException("There is already a compile job in progress!"); } compiling = true; } using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { async Task ProgressTask() { if (!estimatedDuration.HasValue) { return; } progressReporter(0); var ct = progressCts.Token; var sleepInterval = estimatedDuration.Value / 100; try { for (var I = 0; I < 99; ++I) { await Task.Delay(sleepInterval, progressCts.Token).ConfigureAwait(false); progressReporter(I + 1); } } catch (OperationCanceledException) { } } var progressTask = ProgressTask(); try { var commitInsert = revisionInformation.CommitSha.Substring(0, 7); string remoteCommitInsert; if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha) { commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert); remoteCommitInsert = String.Empty; } else { remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7)); } var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})", String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => { var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7)); if (x.Comment != null) { result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment); } return(result); }))); using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false)) { await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false); async Task CleanupFailedCompile(bool cancelled) { logger.LogTrace("Cleaning compile directory..."); var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken); try { await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false); } catch (Exception e) { logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e); } await chatTask.ConfigureAwait(false); }; try { await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false); var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName); logger.LogTrace("Copying repository to game directory..."); //copy the repository var fullDirA = ioManager.ResolvePath(dirA); var repoOrigin = repository.Origin; using (repository) await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false); //run precompile scripts var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)); await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> { resolvedGameDirectory, repoOrigin }, cancellationToken).ConfigureAwait(false); //determine the dme if (job.DmeName == null) { logger.LogTrace("Searching for available .dmes..."); var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); if (path == default) { throw new JobException("Unable to find any .dme!"); } var dmeWithExtension = ioManager.GetFileName(path); job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1); } else if (!await ioManager.FileExists(ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)), cancellationToken).ConfigureAwait(false)) { throw new JobException("Unable to locate specified .dme!"); } logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName); await ModifyDme(job, cancellationToken).ConfigureAwait(false); //run compiler, verify api job.ByondVersion = byondLock.Version.ToString(); var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false); try { if (exitCode != 0) { throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output)); } await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false); } catch (JobException) { //server never validated or compile failed await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> { resolvedGameDirectory, exitCode == 0 ? "1" : "0" }, cancellationToken).ConfigureAwait(false); throw; } logger.LogTrace("Running post compile event..."); await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)) }, cancellationToken).ConfigureAwait(false); logger.LogTrace("Duplicating compiled game..."); //duplicate the dmb et al await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); logger.LogTrace("Applying static game file symlinks..."); //symlink in the static data var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken); var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false); logger.LogDebug("Compile complete!"); return(job); } catch (Exception e) { await CleanupFailedCompile(e is OperationCanceledException).ConfigureAwait(false); throw; } } } catch (OperationCanceledException) { await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false); throw; } finally { compiling = false; progressCts.Cancel(); await progressTask.ConfigureAwait(false); } } }
/// <inheritdoc /> public Task Initialize(CancellationToken cancellationToken) { return(Task.Factory.StartNew(async() => { using (logger.BeginScope("Initializing repository...")) { var repoPath = ioManager.ResolvePath(ioManager.ConcatPath(gitHubConfiguration.RepoOwner, gitHubConfiguration.RepoName)); logger.LogTrace("Repo path evaluated to be: {0}", repoPath); try { logger.LogTrace("Creating repository object."); cancellationToken.ThrowIfCancellationRequested(); repositoryObject = new LibGit2Sharp.Repository(repoPath); repositoryObject.RemoveUntrackedFiles(); cancellationToken.ThrowIfCancellationRequested(); repositoryObject.RetrieveStatus(); } catch (OperationCanceledException e) { logger.LogDebug(e, "Repository setup cancelled!"); repositoryObject?.Dispose(); throw; } catch (Exception e) { cancellationToken.ThrowIfCancellationRequested(); using (logger.BeginScope("Repository fallback initializing...")) { repositoryObject?.Dispose(); try { logger.LogTrace("Checking repository directory exists."); if (await ioManager.DirectoryExists(repoPath, cancellationToken).ConfigureAwait(false)) { logger.LogWarning(e, "Failed to load repository! Deleting and cloning..."); await ioManager.DeleteDirectory(repoPath, cancellationToken).ConfigureAwait(false); } else { logger.LogInformation(e, "Cloning repository..."); } LibGit2Sharp.Repository.Clone(String.Format(CultureInfo.InvariantCulture, "https://github.com/{0}/{1}", gitHubConfiguration.RepoOwner, gitHubConfiguration.RepoName), repoPath, new CloneOptions { Checkout = false, RecurseSubmodules = true, OnProgress = (a) => !cancellationToken.IsCancellationRequested, OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, OnTransferProgress = (a) => !cancellationToken.IsCancellationRequested }); logger.LogInformation("Repo clone completed."); repositoryObject = new LibGit2Sharp.Repository(repoPath); } catch (UserCancelledException e2) { logger.LogDebug(e2, "Repository setup cancelled!"); cancellationToken.ThrowIfCancellationRequested(); } catch (Exception e2) { logger.LogCritical(e2, "Unable to clone repository!"); throw; } } } } }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current)); }
#pragma warning restore CA1506 /// <inheritdoc /> #pragma warning disable CA1506 // TODO: Decomplexify public async Task CleanUnusedCompileJobs(CancellationToken cancellationToken) { List <long> jobIdsToSkip; // don't clean locked directories lock (jobLockCounts) jobIdsToSkip = jobLockCounts.Select(x => x.Key).ToList(); List <string> jobUidsToNotErase = null; // find the uids of locked directories await databaseContextFactory.UseContext(async db => { jobUidsToNotErase = (await db .CompileJobs .AsQueryable() .Where( x => x.Job.Instance.Id == metadata.Id && jobIdsToSkip.Contains(x.Id.Value)) .Select(x => x.DirectoryName.Value) .ToListAsync(cancellationToken) .ConfigureAwait(false)) .Select(x => x.ToString()) .ToList(); }).ConfigureAwait(false); jobUidsToNotErase.Add(SwappableDmbProvider.LiveGameDirectory); logger.LogTrace("We will not clean the following directories: {0}", String.Join(", ", jobUidsToNotErase)); // cleanup var gameDirectory = ioManager.ResolvePath(); await ioManager.CreateDirectory(gameDirectory, cancellationToken).ConfigureAwait(false); var directories = await ioManager.GetDirectories(gameDirectory, cancellationToken).ConfigureAwait(false); int deleting = 0; var tasks = directories.Select(async x => { var nameOnly = ioManager.GetFileName(x); if (jobUidsToNotErase.Contains(nameOnly)) { return; } logger.LogDebug("Cleaning unused game folder: {0}...", nameOnly); try { ++deleting; await ioManager.DeleteDirectory(x, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { throw; } catch (Exception e) { logger.LogWarning(e, "Error deleting directory {0}!", x); } }).ToList(); if (deleting > 0) { await Task.WhenAll(tasks).ConfigureAwait(false); } }
/// <inheritdoc /> public async Task <IRepository> CloneRepository(Uri url, string initialBranch, string username, string password, Action <int> progressReporter, CancellationToken cancellationToken) { if (url == null) { throw new ArgumentNullException(nameof(url)); } if (progressReporter == null) { throw new ArgumentNullException(nameof(progressReporter)); } logger.LogInformation("Begin clone {0} (Branch: {1})", url, initialBranch); lock (semaphore) { if (CloneInProgress) { throw new JobException(ErrorCode.RepoCloning); } CloneInProgress = true; } try { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { logger.LogTrace("Semaphore acquired"); var repositoryPath = ioManager.ResolvePath(); if (!await ioManager.DirectoryExists(repositoryPath, cancellationToken).ConfigureAwait(false)) { try { var cloneOptions = new CloneOptions { OnProgress = (a) => !cancellationToken.IsCancellationRequested, OnTransferProgress = (a) => { var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); progressReporter((int)percentage); return(!cancellationToken.IsCancellationRequested); }, RecurseSubmodules = true, OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested, BranchName = initialBranch, CredentialsProvider = repositoryFactory.GenerateCredentialsHandler(username, password) }; await repositoryFactory.Clone( url, cloneOptions, repositoryPath, cancellationToken) .ConfigureAwait(false); } catch { try { logger.LogTrace("Deleting partially cloned repository..."); await ioManager.DeleteDirectory(repositoryPath, default).ConfigureAwait(false); } catch (Exception e) { logger.LogDebug("Error deleting partially cloned repository! Exception: {0}", e); } throw; } } else { logger.LogDebug("Repository exists, clone aborted!"); return(null); } } logger.LogInformation("Clone complete!"); } finally { CloneInProgress = false; } return(await LoadRepository(cancellationToken).ConfigureAwait(false)); }
/// <inheritdoc /> public bool ApplyUpdate(Version version, Uri updateZipUrl, IIOManager ioManager) { if (version == null) { throw new ArgumentNullException(nameof(version)); } if (updateZipUrl == null) { throw new ArgumentNullException(nameof(updateZipUrl)); } if (ioManager == null) { throw new ArgumentNullException(nameof(ioManager)); } CheckSanity(true); logger.LogTrace("Begin ApplyUpdate..."); lock (this) { if (updating || RestartRequested) { logger.LogTrace("Aborted due to concurrency conflict!"); return(false); } updating = true; } async void RunUpdate() { try { logger.LogInformation("Updating server to version {0} ({1})...", version, updateZipUrl); if (cancellationTokenSource == null) { throw new InvalidOperationException("Tried to update a non-running Server!"); } var cancellationToken = cancellationTokenSource.Token; logger.LogTrace("Downloading zip package..."); var updateZipData = await ioManager.DownloadFile(updateZipUrl, cancellationToken).ConfigureAwait(false); try { logger.LogTrace("Exctracting zip package to {0}...", updatePath); await ioManager.ZipToDirectory(updatePath, updateZipData, cancellationToken).ConfigureAwait(false); } catch (Exception e) { updating = false; try { // important to not leave this directory around if possible await ioManager.DeleteDirectory(updatePath, default).ConfigureAwait(false); } catch (Exception e2) { throw new AggregateException(e, e2); } throw; } await Restart(version, null).ConfigureAwait(false); } catch (OperationCanceledException) { logger.LogInformation("Server update cancelled!"); } catch (Exception e) { logger.LogError("Error updating server! Exception: {0}", e); } finally { updating = false; } } RunUpdate(); return(true); }
/// <inheritdoc /> public async Task <IRepository> CloneRepository(Uri url, string initialBranch, string username, string password, Action <int> progressReporter, CancellationToken cancellationToken) { if (url == null) { throw new ArgumentNullException(nameof(url)); } if (progressReporter == null) { throw new ArgumentNullException(nameof(progressReporter)); } logger.LogInformation("Begin clone {0} (Branch: {1})", url, initialBranch); lock (this) { if (CloneInProgress) { throw new InvalidOperationException("The repository is already being cloned!"); } CloneInProgress = true; } try { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { logger.LogTrace("Semaphore acquired"); if (!await ioManager.DirectoryExists(".", cancellationToken).ConfigureAwait(false)) { try { await Task.Factory.StartNew(() => { string path = null; try { path = LibGit2Sharp.Repository.Clone(url.ToString(), ioManager.ResolvePath("."), new CloneOptions { OnProgress = (a) => !cancellationToken.IsCancellationRequested, OnTransferProgress = (a) => { var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); progressReporter((int)percentage); return(!cancellationToken.IsCancellationRequested); }, RecurseSubmodules = true, OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested, BranchName = initialBranch, CredentialsProvider = credentialsProvider.GenerateHandler(username, password) }); } catch (UserCancelledException) { } cancellationToken.ThrowIfCancellationRequested(); }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); } catch { try { logger.LogTrace("Deleting partially cloned repository..."); await ioManager.DeleteDirectory(".", default).ConfigureAwait(false); } catch (Exception e) { logger.LogDebug("Error deleting partially cloned repository! Exception: {0}", e); } throw; } } else { logger.LogDebug("Repository exists, clone aborted!"); return(null); } } logger.LogInformation("Clone complete!"); } finally { CloneInProgress = false; } return(await LoadRepository(cancellationToken).ConfigureAwait(false)); }
/// <inheritdoc /> public async Task StartAsync(CancellationToken cancellationToken) { async Task <byte[]> GetActiveVersion() { var activeVersionFileExists = await ioManager.FileExists(ActiveVersionFileName, cancellationToken).ConfigureAwait(false); return(!activeVersionFileExists ? null : await ioManager.ReadAllBytes(ActiveVersionFileName, cancellationToken).ConfigureAwait(false)); } var activeVersionBytesTask = GetActiveVersion(); // Create local cfg directory in case it doesn't exist var localCfgDirectory = ioManager.ConcatPath( byondInstaller.PathToUserByondFolder, CfgDirectoryName); await ioManager.CreateDirectory( localCfgDirectory, cancellationToken).ConfigureAwait(false); // Delete trusted.txt so it doesn't grow too large var trustedFilePath = ioManager.ConcatPath( localCfgDirectory, TrustedDmbFileName); logger.LogTrace("Deleting trusted .dmbs file {0}", trustedFilePath); await ioManager.DeleteFile( trustedFilePath, cancellationToken).ConfigureAwait(false); var byondDirectory = ioManager.ResolvePath(); await ioManager.CreateDirectory(byondDirectory, cancellationToken).ConfigureAwait(false); var directories = await ioManager.GetDirectories(byondDirectory, cancellationToken).ConfigureAwait(false); async Task ReadVersion(string path) { var versionFile = ioManager.ConcatPath(path, VersionFileName); if (!await ioManager.FileExists(versionFile, cancellationToken).ConfigureAwait(false)) { logger.LogInformation("Cleaning unparsable version path: {0}", ioManager.ResolvePath(path)); await ioManager.DeleteDirectory(path, cancellationToken).ConfigureAwait(false); // cleanup return; } var bytes = await ioManager.ReadAllBytes(versionFile, cancellationToken).ConfigureAwait(false); var text = Encoding.UTF8.GetString(bytes); if (Version.TryParse(text, out var version)) { var key = VersionKey(version, true); lock (installedVersions) if (!installedVersions.ContainsKey(key)) { logger.LogDebug("Adding detected BYOND version {0}...", key); installedVersions.Add(key, Task.CompletedTask); return; } } await ioManager.DeleteDirectory(path, cancellationToken).ConfigureAwait(false); } await Task.WhenAll(directories.Select(x => ReadVersion(x))).ConfigureAwait(false); var activeVersionBytes = await activeVersionBytesTask.ConfigureAwait(false); if (activeVersionBytes != null) { var activeVersionString = Encoding.UTF8.GetString(activeVersionBytes); bool hasRequestedActiveVersion; lock (installedVersions) hasRequestedActiveVersion = installedVersions.ContainsKey(activeVersionString); if (hasRequestedActiveVersion && Version.TryParse(activeVersionString, out var activeVersion)) { ActiveVersion = activeVersion.Semver(); } else { logger.LogWarning("Failed to load saved active version {0}!", activeVersionString); await ioManager.DeleteFile(ActiveVersionFileName, cancellationToken).ConfigureAwait(false); } } }