Example #1
0
        /// <summary>
        /// Creates a pre-configured <see cref="Process"/> pointing to dmm-tools.exe
        /// </summary>
        /// <param name="output">A <see cref="StringBuilder"/> used to accept stdout</param>
        /// <param name="errorOutput">A <see cref="StringBuilder"/> used to accept stderr</param>
        /// <returns>A <see cref="Task"/> resulting in a pre-configured <see cref="Process"/> pointing to dmm-tools.exe</returns>
        Process CreateDMMToolsProcess(StringBuilder output, StringBuilder errorOutput)
        {
            var P = new Process();

            P.StartInfo.RedirectStandardOutput = true;
            P.StartInfo.RedirectStandardError  = true;
            P.StartInfo.UseShellExecute        = false;
            P.StartInfo.WorkingDirectory       = ioManager.ResolvePath(".");
            P.OutputDataReceived += new DataReceivedEventHandler(
                delegate(object sender, DataReceivedEventArgs e)
            {
                output.Append(Environment.NewLine);
                output.Append(e.Data);
            }
                );
            P.ErrorDataReceived += new DataReceivedEventHandler(
                delegate(object sender, DataReceivedEventArgs e)
            {
                errorOutput.Append(Environment.NewLine);
                errorOutput.Append(e.Data);
            }
                );

            try
            {
                P.StartInfo.FileName = ioManager.ConcatPath(ioManager.GetDirectoryName(Assembly.GetExecutingAssembly().Location), DMMToolsPath);
            }
            catch
            {
                P.Dispose();
                throw;
            }

            return(P);
        }
        /// <inheritdoc />
        public Task InstallByond(string path, Version version, CancellationToken cancellationToken)
        {
            if (path == null)
            {
                throw new ArgumentNullException(nameof(path));
            }
            if (version == null)
            {
                throw new ArgumentNullException(nameof(version));
            }

            // write the scripts for running the ting
            // need to add $ORIGIN to LD_LIBRARY_PATH
            const string StandardScript = "#!/bin/sh\nexport LD_LIBRARY_PATH=\"\\$ORIGIN:$LD_LIBRARY_PATH\"\nBASEDIR=$(dirname \"$0\")\nexec \"$BASEDIR/{0}\" \"$@\"\n";

            var dreamDaemonScript = String.Format(CultureInfo.InvariantCulture, StandardScript, DreamDaemonExecutableName);
            var dreamMakerScript  = String.Format(CultureInfo.InvariantCulture, StandardScript, DreamMakerExecutableName);

            async Task WriteAndMakeExecutable(string fullPath, string script)
            {
                await ioManager.WriteAllBytes(fullPath, Encoding.ASCII.GetBytes(script), cancellationToken).ConfigureAwait(false);

                postWriteHandler.HandleWrite(fullPath);
            }

            var basePath = ioManager.ConcatPath(path, ByondManager.BinPath);

            var task = Task.WhenAll(WriteAndMakeExecutable(ioManager.ConcatPath(basePath, DreamDaemonName), dreamDaemonScript), WriteAndMakeExecutable(ioManager.ConcatPath(basePath, DreamMakerName), dreamMakerScript));

            postWriteHandler.HandleWrite(ioManager.ConcatPath(basePath, DreamDaemonExecutableName));
            postWriteHandler.HandleWrite(ioManager.ConcatPath(basePath, DreamMakerExecutableName));

            return(task);
        }
Example #3
0
        /// <inheritdoc />
        public async Task <IJsonTrackingContext> TrackJsons(string basePath, string channelsJsonName, string commandsJsonName, CancellationToken cancellationToken)
        {
            if (customCommandHandler == null)
            {
                throw new InvalidOperationException("RegisterCommandHandler() hasn't been called!");
            }
            JsonTrackingContext context = null;

            context = new JsonTrackingContext(ioManager, customCommandHandler, () =>
            {
                lock (trackingContexts)
                    trackingContexts.Remove(context);
            }, ioManager.ConcatPath(basePath, commandsJsonName), ioManager.ConcatPath(basePath, channelsJsonName));
            Task task;

            lock (trackingContexts)
            {
                trackingContexts.Add(context);
                lock (mappedChannels)
                    task = Task.WhenAll(trackingContexts.Select(x => x.SetChannels(mappedChannels.Select(y => y.Value.Channel), cancellationToken)));
            }
            await task.ConfigureAwait(false);

            return(context);
        }
        /// <inheritdoc />
        public async Task <ServerSideModifications> CopyDMFilesTo(string dmeFile, string destination, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
            {
                await EnsureDirectories(cancellationToken).ConfigureAwait(false);

                // just assume no other fs race conditions here
                var dmeExistsTask      = ioManager.FileExists(ioManager.ConcatPath(CodeModificationsSubdirectory, dmeFile), cancellationToken);
                var headFileExistsTask = ioManager.FileExists(ioManager.ConcatPath(CodeModificationsSubdirectory, CodeModificationsHeadFile), cancellationToken);
                var tailFileExistsTask = ioManager.FileExists(ioManager.ConcatPath(CodeModificationsSubdirectory, CodeModificationsTailFile), cancellationToken);
                var copyTask           = ioManager.CopyDirectory(CodeModificationsSubdirectory, destination, null, cancellationToken);

                await Task.WhenAll(dmeExistsTask, headFileExistsTask, tailFileExistsTask, copyTask).ConfigureAwait(false);

                if (!dmeExistsTask.Result && !headFileExistsTask.Result && !tailFileExistsTask.Result)
                {
                    return(null);
                }

                if (dmeExistsTask.Result)
                {
                    return(new ServerSideModifications(null, null, true));
                }

                if (!headFileExistsTask.Result && !tailFileExistsTask.Result)
                {
                    return(null);
                }
Example #5
0
        /// <summary>
        /// Adds server side includes to the .dme being compiled
        /// </summary>
        /// <param name="job">The <see cref="CompileJob"/> for the operation</param>
        /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
        /// <returns>A <see cref="Task"/> representing the running operation</returns>
        async Task ModifyDme(Models.CompileJob job, CancellationToken cancellationToken)
        {
            var dmeFileName = String.Join('.', job.DmeName, DmeExtension);
            var dmePath     = ioManager.ConcatPath(job.DirectoryName.ToString(), dmeFileName);
            var dmeReadTask = ioManager.ReadAllBytes(dmePath, cancellationToken);

            var dmeModificationsTask = configuration.CopyDMFilesTo(dmeFileName, ioManager.ResolvePath(job.DirectoryName.ToString()), cancellationToken);

            var dmeBytes = await dmeReadTask.ConfigureAwait(false);

            var dme = Encoding.UTF8.GetString(dmeBytes);

            var dmeModifications = await dmeModificationsTask.ConfigureAwait(false);

            if (dmeModifications == null || dmeModifications.TotalDmeOverwrite)
            {
                if (dmeModifications != null)
                {
                    logger.LogDebug(".dme replacement configured!");
                }
                else
                {
                    logger.LogTrace("No .dme modifications required.");
                }
                return;
            }

            var dmeLines = new List <string>(dme.Split('\n', StringSplitOptions.None));

            for (var I = 0; I < dmeLines.Count; ++I)
            {
                var line = dmeLines[I];
                if (line.Contains("BEGIN_INCLUDE", StringComparison.Ordinal) && dmeModifications.HeadIncludeLine != null)
                {
                    var headIncludeLineNumber = I + 1;
                    logger.LogDebug(
                        "Inserting HeadInclude.dm at line {0}: {1}",
                        headIncludeLineNumber,
                        dmeModifications.HeadIncludeLine);
                    dmeLines.Insert(headIncludeLineNumber, dmeModifications.HeadIncludeLine);
                    ++I;
                }
                else if (line.Contains("END_INCLUDE", StringComparison.Ordinal) && dmeModifications.TailIncludeLine != null)
                {
                    logger.LogDebug(
                        "Inserting TailInclude.dm at line {0}: {1}",
                        I,
                        dmeModifications.TailIncludeLine);
                    dmeLines.Insert(I, dmeModifications.TailIncludeLine);
                    break;
                }
            }

            dmeBytes = Encoding.UTF8.GetBytes(String.Join(Environment.NewLine, dmeLines));
            await ioManager.WriteAllBytes(dmePath, dmeBytes, cancellationToken).ConfigureAwait(false);
        }
 /// <inheritdoc />
 public async Task CleanCache(CancellationToken cancellationToken)
 {
     try
     {
         await ioManager.DeleteDirectory(ioManager.ConcatPath(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "byond/cache"), cancellationToken).ConfigureAwait(false);
     }
     catch (Exception e)
     {
         logger.LogWarning("Error deleting BYOND cache! Exception: {0}", e);
     }
 }
        /// <inheritdoc />
        public async Task <IByondExecutableLock> UseExecutables(Version requiredVersion, CancellationToken cancellationToken)
        {
            var versionToUse = requiredVersion ?? ActiveVersion;

            if (versionToUse == null)
            {
                throw new JobException(ErrorCode.ByondNoVersionsInstalled);
            }
            await InstallVersion(versionToUse, null, cancellationToken).ConfigureAwait(false);

            var versionKey        = VersionKey(versionToUse, true);
            var binPathForVersion = ioManager.ConcatPath(versionKey, BinPath);

            logger.LogTrace("Creating ByondExecutableLock lock for version {0}", versionToUse);
            return(new ByondExecutableLock(
                       ioManager,
                       semaphore,
                       versionToUse,
                       ioManager.ResolvePath(
                           ioManager.ConcatPath(
                               binPathForVersion,
                               byondInstaller.DreamDaemonName)),
                       ioManager.ResolvePath(
                           ioManager.ConcatPath(
                               binPathForVersion,
                               byondInstaller.DreamMakerName)),
                       ioManager.ResolvePath(
                           ioManager.ConcatPath(
                               byondInstaller.PathToUserByondFolder,
                               CfgDirectoryName,
                               TrustedDmbFileName))));
        }
Example #8
0
        /// <inheritdoc />
        public async Task <IDmbProvider> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken)
        {
            logger.LogTrace("Loading compile job {0}...", compileJob.Id);
            var providerSubmitted = false;
            var newProvider       = new DmbProvider(compileJob, ioManager, () =>
            {
                if (providerSubmitted)
                {
                    CleanJob(compileJob);
                }
            });

            try
            {
                var primaryCheckTask   = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken);
                var secondaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken);

                if (!(await primaryCheckTask.ConfigureAwait(false) && await secondaryCheckTask.ConfigureAwait(false)))
                {
                    logger.LogWarning("Error loading compile job, .dmb missing!");
                    return(null);                       //omae wa mou shinderu
                }

                lock (this)
                {
                    if (!jobLockCounts.TryGetValue(compileJob.Id, out int value))
                    {
                        value = 1;
                        jobLockCounts.Add(compileJob.Id, 1);
                    }
                    else
                    {
                        jobLockCounts[compileJob.Id] = ++value;
                    }

                    logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value);

                    providerSubmitted = true;
                    return(newProvider);
                }
            }
            finally
            {
                if (!providerSubmitted)
                {
                    newProvider.Dispose();
                }
            }
        }
        /// <summary>
        /// Gets the evaluated log <see cref="Directory"/>.
        /// </summary>
        /// <param name="ioManager">The <see cref="IIOManager"/> to use.</param>
        /// <param name="assemblyInformationProvider">The <see cref="IAssemblyInformationProvider"/> to use.</param>
        /// <param name="platformIdentifier">The <see cref="IPlatformIdentifier"/> to use</param>
        /// <returns>The evaluated log <see cref="Directory"/>.</returns>
        public string GetFullLogDirectory(
            IIOManager ioManager,
            IAssemblyInformationProvider assemblyInformationProvider,
            IPlatformIdentifier platformIdentifier)
        {
            if (ioManager == null)
            {
                throw new ArgumentNullException(nameof(ioManager));
            }
            if (assemblyInformationProvider == null)
            {
                throw new ArgumentNullException(nameof(assemblyInformationProvider));
            }
            if (platformIdentifier == null)
            {
                throw new ArgumentNullException(nameof(platformIdentifier));
            }

            var directoryToUse = platformIdentifier.IsWindows
                                ? Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData) // C:/ProgramData
                                : "/var/log";                                                                // :pain:

            return(!String.IsNullOrEmpty(Directory)
                                ? Directory
                                : ioManager.ConcatPath(
                       directoryToUse,
                       assemblyInformationProvider.VersionPrefix));
        }
Example #10
0
        public Task <IActionResult> ListLogs([FromQuery] int?page, [FromQuery] int?pageSize, CancellationToken cancellationToken)
        => Paginated(
            async() =>
        {
            var path = fileLoggingConfiguration.GetFullLogDirectory(ioManager, assemblyInformationProvider, platformIdentifier);
            try
            {
                var files = await ioManager.GetFiles(path, cancellationToken).ConfigureAwait(false);
                var tasks = files.Select(
                    async file => new LogFileResponse
                {
                    Name         = ioManager.GetFileName(file),
                    LastModified = await ioManager
                                   .GetLastModified(
                        ioManager.ConcatPath(path, file),
                        cancellationToken)
                                   .ConfigureAwait(false),
                })
                            .ToList();

                await Task.WhenAll(tasks).ConfigureAwait(false);

                return(new PaginatableResult <LogFileResponse>(
                           tasks
                           .AsQueryable()
                           .Select(x => x.Result)
                           .OrderByDescending(x => x.Name)));
            }
            catch (IOException ex)
            {
                return(new PaginatableResult <LogFileResponse>(
                           Conflict(new ErrorMessageResponse(ErrorCode.IOError)
                {
                    AdditionalData = ex.ToString(),
                })));
            }
        },
            null,
            page,
            pageSize,
            cancellationToken);
Example #11
0
        public async Task <IActionResult> ListLogs(CancellationToken cancellationToken)
        {
            var path = fileLoggingConfiguration.GetFullLogDirectory(ioManager, assemblyInformationProvider, platformIdentifier);

            try
            {
                var files = await ioManager.GetFiles(path, cancellationToken).ConfigureAwait(false);

                var tasks = files.Select(
                    async file => new LogFile
                {
                    Name         = ioManager.GetFileName(file),
                    LastModified = await ioManager.GetLastModified(
                        ioManager.ConcatPath(path, file),
                        cancellationToken)
                                   .ConfigureAwait(false)
                })
                            .ToList();

                await Task.WhenAll(tasks).ConfigureAwait(false);

                var result = tasks
                             .Select(x => x.Result)
                             .OrderByDescending(x => x.Name)
                             .ToList();

                return(Ok(result));
            }
            catch (IOException ex)
            {
                return(Conflict(new ErrorMessage(ErrorCode.IOError)
                {
                    AdditionalData = ex.ToString()
                }));
            }
        }
        /// <inheritdoc />
        public async Task <string> ModifyMerge(PullRequest pullRequest, string workingCommit, CancellationToken cancellationToken)
        {
            if (pullRequest == null)
            {
                throw new ArgumentNullException(nameof(pullRequest));
            }
            if (workingCommit == null)
            {
                throw new ArgumentNullException(nameof(workingCommit));
            }

            var changelog = Changelog.GetChangelog(pullRequest, out bool malformed);

            if (changelog == null)
            {
                return(workingCommit);
            }

            var result = new List <Dictionary <string, string> >();

            result.AddRange(changelog.Changes.Select(x => new Dictionary <string, string> {
                { x.Type.ToString().ToLowerInvariant(), x.Text }
            }));

            //create the object graph
            var graph = new
            {
                author = changelog.Author,
                delete_after_temporary_for_replacement = true,
                changes = result
            };
            //hack because '-' isn't a valid identifier in c#
            var yaml = new Serializer().Serialize(graph).Replace("delete_after_temporary_for_replacement", "delete-after");

            var title = String.Format(CultureInfo.InvariantCulture, "AutoChangeLog-pr-{0}.yml", pullRequest.Number);

            var pathToWrite = ioManager.ConcatPath(repository.Path, "html", "changelogs", title);
            await ioManager.WriteAllText(pathToWrite, yaml, cancellationToken).ConfigureAwait(false);

            return(await repository.CommitChanges(new List <string> {
                pathToWrite
            }, cancellationToken));
        }
        /// <inheritdoc />
        public async Task <ILocalRepository> GetRepository(Octokit.Repository repository, Func <int, Task> onCloneProgress, Func <Task> onOperationBlocked, CancellationToken cancellationToken)
        {
            if (repository == null)
            {
                throw new ArgumentNullException(nameof(repository));
            }

            var repoPath = ioManager.ConcatPath(repository.Owner.Login, repository.Name);

            TaskCompletionSource <object> usageTask = null;

            try
            {
                return(await TryLoadRepository(repoPath, onOperationBlocked, tcs => usageTask = tcs, cancellationToken).ConfigureAwait(false));
            }
            catch (LibGit2SharpException e)
            {
                logger.LogWarning(e, "Failed to load repository {0}/{1}! Cloning...", repository.Owner.Login, repository.Name);
            }

            //so the repo failed to load and now we're holding our queue spot in usageTask
            //reclone it
            try
            {
                await ioManager.DeleteDirectory(repoPath, cancellationToken).ConfigureAwait(false);

                await ioManager.CreateDirectory(repoPath, cancellationToken).ConfigureAwait(false);

                await repositoryOperations.Clone(repository.CloneUrl, ioManager.ResolvePath(repoPath), onCloneProgress, cancellationToken).ConfigureAwait(false);

                return(await localRepositoryFactory.CreateLocalRepository(ioManager.ResolvePath(repoPath), usageTask, cancellationToken).ConfigureAwait(false));
            }
            catch
            {
                //ok we can't do anything else, clear our queue spot
                usageTask.SetResult(null);
                throw;
            }
        }
Example #14
0
        /// <summary>
        /// Run a quick DD instance to test the DMAPI is installed on the target code
        /// </summary>
        /// <param name="timeout">The timeout in seconds for validation</param>
        /// <param name="securityLevel">The <see cref="DreamDaemonSecurity"/> level to use to validate the API</param>
        /// <param name="job">The <see cref="Models.CompileJob"/> for the operation</param>
        /// <param name="byondLock">The current <see cref="IByondExecutableLock"/></param>
        /// <param name="portToUse">The port to use for API validation</param>
        /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
        /// <returns>A <see cref="Task{TResult}"/> resulting in <see langword="true"/> if the DMAPI was successfully validated, <see langword="false"/> otherwise</returns>
        async Task <bool> VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, ushort portToUse, CancellationToken cancellationToken)
        {
            logger.LogTrace("Verifying DMAPI...");
            var launchParameters = new DreamDaemonLaunchParameters
            {
                AllowWebClient = false,
                PrimaryPort    = portToUse,
                SecurityLevel  = securityLevel,                   //all it needs to read the file and exit
                StartupTimeout = timeout
            };

            var dirA     = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
            var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Concat(job.DmeName, DmbExtension), job);

            var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout);

            using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false))
            {
                var launchResult = await controller.LaunchResult.ConfigureAwait(false);

                var now = DateTimeOffset.Now;
                if (now < timeoutAt && launchResult.StartupTime.HasValue)
                {
                    var timeoutTask = Task.Delay(timeoutAt - now, cancellationToken);

                    await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false);

                    cancellationToken.ThrowIfCancellationRequested();
                }

                if (!controller.Lifetime.IsCompleted)
                {
                    logger.LogDebug("API validation timed out!");
                    return(false);
                }

                var validated = controller.ApiValidated;
                logger.LogTrace("API valid: {0}", validated);
                return(validated);
            }
        }
Example #15
0
        /// <inheritdoc />
        public async Task <ServerSideModifications> CopyDMFilesTo(string dmeFile, string destination, CancellationToken cancellationToken)
        {
            await EnsureDirectories(cancellationToken).ConfigureAwait(false);

            //just assume no other fs race conditions here
            var dmeExistsTask      = ioManager.FileExists(ioManager.ConcatPath(CodeModificationsSubdirectory, dmeFile), cancellationToken);
            var headFileExistsTask = ioManager.FileExists(ioManager.ConcatPath(CodeModificationsSubdirectory, CodeModificationsHeadFile), cancellationToken);
            var tailFileExistsTask = ioManager.FileExists(ioManager.ConcatPath(CodeModificationsSubdirectory, CodeModificationsTailFile), cancellationToken);

            await Task.WhenAll(dmeExistsTask, headFileExistsTask, tailFileExistsTask).ConfigureAwait(false);

            if (!dmeExistsTask.Result && !headFileExistsTask.Result && !tailFileExistsTask.Result)
            {
                return(null);
            }

            var copyTask = ioManager.CopyDirectory(CodeModificationsSubdirectory, destination, null, cancellationToken);

            if (dmeExistsTask.Result)
            {
                await copyTask.ConfigureAwait(false);

                return(new ServerSideModifications(null, null, true));
            }

            if (!headFileExistsTask.Result && !tailFileExistsTask.Result)
            {
                await copyTask.ConfigureAwait(false);

                return(null);
            }

            string IncludeLine(string filePath) => String.Format(CultureInfo.InvariantCulture, "#include \"{0}\"", filePath);

            await copyTask.ConfigureAwait(false);

            return(new ServerSideModifications(headFileExistsTask.Result ? IncludeLine(CodeModificationsHeadFile) : null, tailFileExistsTask.Result ? IncludeLine(CodeModificationsTailFile) : null, false));
        }
        /// <inheritdoc />
                #pragma warning disable CA1506 // TODO: Decomplexify
        public async Task <ISessionController> LaunchNew(DreamDaemonLaunchParameters launchParameters, IDmbProvider dmbProvider, IByondExecutableLock currentByondLock, bool primaryPort, bool primaryDirectory, bool apiValidate, CancellationToken cancellationToken)
        {
            var portToUse = primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort;

            if (!portToUse.HasValue)
            {
                throw new InvalidOperationException("Given port is null!");
            }
            var accessIdentifier = cryptographySuite.GetSecureString();

            const string JsonPostfix = "tgs.json";

            var basePath = primaryDirectory ? dmbProvider.PrimaryDirectory : dmbProvider.SecondaryDirectory;

            // delete all previous tgs json files
            var files = await ioManager.GetFilesWithExtension(basePath, JsonPostfix, cancellationToken).ConfigureAwait(false);

            await Task.WhenAll(files.Select(x => ioManager.DeleteFile(x, cancellationToken))).ConfigureAwait(false);

            // i changed this back from guids, hopefully i don't regret that
            string JsonFile(string name) => String.Format(CultureInfo.InvariantCulture, "{0}.{1}", name, JsonPostfix);

            var securityLevelToUse = launchParameters.SecurityLevel.Value;

            switch (dmbProvider.CompileJob.MinimumSecurityLevel)
            {
            case DreamDaemonSecurity.Ultrasafe:
                break;

            case DreamDaemonSecurity.Safe:
                if (securityLevelToUse == DreamDaemonSecurity.Ultrasafe)
                {
                    securityLevelToUse = DreamDaemonSecurity.Safe;
                }
                break;

            case DreamDaemonSecurity.Trusted:
                securityLevelToUse = DreamDaemonSecurity.Trusted;
                break;

            default:
                throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Invalid DreamDaemonSecurity value: {0}", dmbProvider.CompileJob.MinimumSecurityLevel));
            }

            // setup interop files
            var interopInfo = new JsonFile
            {
                AccessIdentifier   = accessIdentifier,
                ApiValidateOnly    = apiValidate,
                ChatChannelsJson   = JsonFile("chat_channels"),
                ChatCommandsJson   = JsonFile("chat_commands"),
                ServerCommandsJson = JsonFile("server_commands"),
                InstanceName       = instance.Name,
                SecurityLevel      = securityLevelToUse,
                Revision           = new Api.Models.Internal.RevisionInformation
                {
                    CommitSha       = dmbProvider.CompileJob.RevisionInformation.CommitSha,
                    OriginCommitSha = dmbProvider.CompileJob.RevisionInformation.OriginCommitSha
                }
            };

            interopInfo.TestMerges.AddRange(dmbProvider.CompileJob.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => new Interop.TestMerge(x, interopInfo.Revision)));

            var interopJsonFile = JsonFile("interop");

            var interopJson = JsonConvert.SerializeObject(interopInfo, new JsonSerializerSettings
            {
                ContractResolver      = new CamelCasePropertyNamesContractResolver(),
                ReferenceLoopHandling = ReferenceLoopHandling.Ignore
            });

            var chatJsonTrackingTask = chat.TrackJsons(basePath, interopInfo.ChatChannelsJson, interopInfo.ChatCommandsJson, cancellationToken);

            await ioManager.WriteAllBytes(ioManager.ConcatPath(basePath, interopJsonFile), Encoding.UTF8.GetBytes(interopJson), cancellationToken).ConfigureAwait(false);

            var chatJsonTrackingContext = await chatJsonTrackingTask.ConfigureAwait(false);

            try
            {
                // get the byond lock
                var byondLock = currentByondLock ?? await byond.UseExecutables(Version.Parse(dmbProvider.CompileJob.ByondVersion), cancellationToken).ConfigureAwait(false);

                try
                {
                    // create interop context
                    var context = new CommContext(ioManager, loggerFactory.CreateLogger <CommContext>(), basePath, interopInfo.ServerCommandsJson);
                    try
                    {
                        // set command line options
                        // more sanitization here cause it uses the same scheme
                        var parameters = String.Format(CultureInfo.InvariantCulture, "{2}={0}&{3}={1}", byondTopicSender.SanitizeString(application.Version.ToString()), byondTopicSender.SanitizeString(interopJsonFile), byondTopicSender.SanitizeString(Constants.DMParamHostVersion), byondTopicSender.SanitizeString(Constants.DMParamInfoJson));

                        var visibility = apiValidate ? "invisible" : "public";

                        // important to run on all ports to allow port changing
                        var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} -ports 1-65535 {2}-close -{3} -{5} -public -params \"{4}\"",
                                                      dmbProvider.DmbName,
                                                      primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort,
                                                      launchParameters.AllowWebClient.Value ? "-webclient " : String.Empty,
                                                      SecurityWord(securityLevelToUse),
                                                      parameters,
                                                      visibility);

                        // See https://github.com/tgstation/tgstation-server/issues/719
                        var noShellExecute = !platformIdentifier.IsWindows;

                        // launch dd
                        var process = processExecutor.LaunchProcess(byondLock.DreamDaemonPath, basePath, arguments, noShellExecute: noShellExecute);
                        try
                        {
                            networkPromptReaper.RegisterProcess(process);

                            // return the session controller for it
                            var result = new SessionController(new ReattachInformation
                            {
                                AccessIdentifier = accessIdentifier,
                                Dmb                = dmbProvider,
                                IsPrimary          = primaryDirectory,
                                Port               = portToUse.Value,
                                ProcessId          = process.Id,
                                ChatChannelsJson   = interopInfo.ChatChannelsJson,
                                ChatCommandsJson   = interopInfo.ChatCommandsJson,
                                ServerCommandsJson = interopInfo.ServerCommandsJson,
                            }, process, byondLock, byondTopicSender, chatJsonTrackingContext, context, chat, loggerFactory.CreateLogger <SessionController>(), launchParameters.SecurityLevel, launchParameters.StartupTimeout);

                            // writeback launch parameter's fixed security level
                            launchParameters.SecurityLevel = securityLevelToUse;

                            return(result);
                        }
                        catch
                        {
                            process.Dispose();
                            throw;
                        }
                    }
                    catch
                    {
                        context.Dispose();
                        throw;
                    }
                }
                catch
                {
                    if (currentByondLock == null)
                    {
                        byondLock.Dispose();
                    }
                    throw;
                }
            }
            catch
            {
                chatJsonTrackingContext.Dispose();
                throw;
            }
        }
 /// <summary>
 /// Get the proper path to <see cref="StaticIgnoreFile"/>
 /// </summary>
 /// <returns>The <see cref="ioManager"/> relative path to <see cref="StaticIgnoreFile"/></returns>
 string StaticIgnorePath() => ioManager.ConcatPath(GameStaticFilesSubdirectory, StaticIgnoreFile);
Example #18
0
        /// <inheritdoc />
                #pragma warning disable CA1506 // TODO: Decomplexify
        public async Task <IDmbProvider> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken)
        {
            if (compileJob == null)
            {
                throw new ArgumentNullException(nameof(compileJob));
            }

            // ensure we have the entire compile job tree
            await databaseContextFactory.UseContext(async db => compileJob = await db.CompileJobs.Where(x => x.Id == compileJob.Id)
                                                    .Include(x => x.Job).ThenInclude(x => x.StartedBy)
                                                    .Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge).ThenInclude(x => x.MergedBy)
                                                    .Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy)
                                                    .FirstAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); // can't wait to see that query

            logger.LogTrace("Loading compile job {0}...", compileJob.Id);
            var providerSubmitted = false;
            var newProvider       = new DmbProvider(compileJob, ioManager, () =>
            {
                if (providerSubmitted)
                {
                    CleanJob(compileJob);
                }
            });

            try
            {
                var primaryCheckTask   = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken);
                var secondaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken);

                if (!(await primaryCheckTask.ConfigureAwait(false) && await secondaryCheckTask.ConfigureAwait(false)))
                {
                    logger.LogWarning("Error loading compile job, .dmb missing!");
                    return(null);                    // omae wa mou shinderu
                }

                lock (this)
                {
                    if (!jobLockCounts.TryGetValue(compileJob.Id, out int value))
                    {
                        value = 1;
                        jobLockCounts.Add(compileJob.Id, 1);
                    }
                    else
                    {
                        jobLockCounts[compileJob.Id] = ++value;
                    }

                    logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value);

                    providerSubmitted = true;
                    return(newProvider);
                }
            }
            finally
            {
                if (!providerSubmitted)
                {
                    newProvider.Dispose();
                }
            }
        }
Example #19
0
        /// <summary>
        /// Installs a BYOND <paramref name="version"/> if it isn't already
        /// </summary>
        /// <param name="version">The BYOND <see cref="Version"/> to install</param>
        /// <param name="customVersionStream">Custom zip file <see cref="Stream"/> to use. Will cause a <see cref="Version.Build"/> number to be added.</param>
        /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
        /// <returns>A <see cref="Task"/> representing the running operation</returns>
        async Task <string> InstallVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken)
        {
            var    ourTcs = new TaskCompletionSource <object>();
            Task   inProgressTask;
            string versionKey;
            bool   installed;

            lock (installedVersions)
            {
                if (customVersionStream != null)
                {
                    int customInstallationNumber = 1;
                    do
                    {
                        versionKey = $"{VersionKey(version, false)}.{customInstallationNumber++}";
                    }while (installedVersions.ContainsKey(versionKey));
                }
                else
                {
                    versionKey = VersionKey(version, true);
                }

                installed = installedVersions.TryGetValue(versionKey, out inProgressTask);
                if (!installed)
                {
                    installedVersions.Add(versionKey, ourTcs.Task);
                }
            }

            if (installed)
            {
                using (cancellationToken.Register(() => ourTcs.SetCanceled()))
                {
                    await Task.WhenAny(ourTcs.Task, inProgressTask).ConfigureAwait(false);

                    cancellationToken.ThrowIfCancellationRequested();
                    return(versionKey);
                }
            }

            if (customVersionStream != null)
            {
                logger.LogInformation("Installing custom BYOND version as {0}...", versionKey);
            }
            else if (version.Build > 0)
            {
                throw new JobException(ErrorCode.ByondNonExistentCustomVersion);
            }
            else
            {
                logger.LogDebug("Requested BYOND version {0} not currently installed. Doing so now...");
            }

            // okay up to us to install it then
            try
            {
                await eventConsumer.HandleEvent(EventType.ByondInstallStart, new List <string> {
                    versionKey
                }, cancellationToken).ConfigureAwait(false);

                var extractPath = ioManager.ResolvePath(versionKey);
                async Task DirectoryCleanup()
                {
                    await ioManager.DeleteDirectory(extractPath, cancellationToken).ConfigureAwait(false);

                    await ioManager.CreateDirectory(extractPath, cancellationToken).ConfigureAwait(false);
                }

                var directoryCleanupTask = DirectoryCleanup();
                try
                {
                    Stream versionZipStream;
                    Stream downloadedStream = null;
                    if (customVersionStream == null)
                    {
                        var bytes = await byondInstaller.DownloadVersion(version, cancellationToken).ConfigureAwait(false);

                        downloadedStream = new MemoryStream(bytes);
                        versionZipStream = downloadedStream;
                    }
                    else
                    {
                        versionZipStream = customVersionStream;
                    }

                    using (downloadedStream)
                    {
                        await directoryCleanupTask.ConfigureAwait(false);

                        logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath);
                        await ioManager.ZipToDirectory(extractPath, versionZipStream, cancellationToken).ConfigureAwait(false);
                    }

                    await byondInstaller.InstallByond(extractPath, version, cancellationToken).ConfigureAwait(false);

                    // make sure to do this last because this is what tells us we have a valid version in the future
                    await ioManager.WriteAllBytes(
                        ioManager.ConcatPath(versionKey, VersionFileName),
                        Encoding.UTF8.GetBytes(versionKey),
                        cancellationToken)
                    .ConfigureAwait(false);
                }
                catch (WebException e)
                {
                    // since the user can easily provide non-exitent version numbers, we'll turn this into a JobException
                    throw new JobException(ErrorCode.ByondDownloadFail, e);
                }
                catch (OperationCanceledException)
                {
                    throw;
                }
                catch
                {
                    await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false);

                    throw;
                }

                ourTcs.SetResult(null);
            }
            catch (Exception e)
            {
                if (!(e is OperationCanceledException))
                {
                    await eventConsumer.HandleEvent(EventType.ByondInstallFail, new List <string> {
                        e.Message
                    }, cancellationToken).ConfigureAwait(false);
                }
                lock (installedVersions)
                    installedVersions.Remove(versionKey);
                ourTcs.SetException(e);
                throw;
            }

            return(versionKey);
        }
Example #20
0
        /// <summary>
        /// Prompts the user to create a <see cref="FileLoggingConfiguration"/>
        /// </summary>
        /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
        /// <returns>A <see cref="Task{TResult}"/> resulting in the new <see cref="FileLoggingConfiguration"/></returns>
        async Task <FileLoggingConfiguration> ConfigureLogging(CancellationToken cancellationToken)
        {
            var fileLoggingConfiguration = new FileLoggingConfiguration();
            await console.WriteAsync(null, true, cancellationToken).ConfigureAwait(false);

            fileLoggingConfiguration.Disable = !await PromptYesNo("Enable file logging? (y/n): ", cancellationToken).ConfigureAwait(false);

            if (!fileLoggingConfiguration.Disable)
            {
                do
                {
                    await console.WriteAsync("Log file directory path (leave blank for default): ", false, cancellationToken).ConfigureAwait(false);

                    fileLoggingConfiguration.Directory = await console.ReadLineAsync(false, cancellationToken).ConfigureAwait(false);

                    if (String.IsNullOrWhiteSpace(fileLoggingConfiguration.Directory))
                    {
                        fileLoggingConfiguration.Directory = null;
                        break;
                    }
                    //test a write of it
                    await console.WriteAsync(null, true, cancellationToken).ConfigureAwait(false);

                    await console.WriteAsync("Testing directory access...", true, cancellationToken).ConfigureAwait(false);

                    try
                    {
                        await ioManager.CreateDirectory(fileLoggingConfiguration.Directory, cancellationToken).ConfigureAwait(false);

                        var testFile = ioManager.ConcatPath(fileLoggingConfiguration.Directory, String.Format(CultureInfo.InvariantCulture, "WizardAccesTest.{0}.deleteme", Guid.NewGuid()));
                        await ioManager.WriteAllBytes(testFile, Array.Empty <byte>(), cancellationToken).ConfigureAwait(false);

                        try
                        {
                            await ioManager.DeleteFile(testFile, cancellationToken).ConfigureAwait(false);
                        }
                        catch (OperationCanceledException)
                        {
                            throw;
                        }
                        catch (Exception e)
                        {
                            await console.WriteAsync(String.Format(CultureInfo.InvariantCulture, "Error deleting test log file: {0}", testFile), true, cancellationToken).ConfigureAwait(false);

                            await console.WriteAsync(e.Message, true, cancellationToken).ConfigureAwait(false);

                            await console.WriteAsync(null, true, cancellationToken).ConfigureAwait(false);
                        }
                        break;
                    }
                    catch (OperationCanceledException)
                    {
                        throw;
                    }
                    catch (Exception e)
                    {
                        await console.WriteAsync(e.Message, true, cancellationToken).ConfigureAwait(false);

                        await console.WriteAsync(null, true, cancellationToken).ConfigureAwait(false);

                        await console.WriteAsync("Please verify the path is valid and you have access to it!", true, cancellationToken).ConfigureAwait(false);
                    }
                } while (true);

                async Task <LogLevel?> PromptLogLevel(string question)
                {
                    do
                    {
                        await console.WriteAsync(null, true, cancellationToken).ConfigureAwait(false);

                        await console.WriteAsync(question, true, cancellationToken).ConfigureAwait(false);

                        await console.WriteAsync(String.Format(CultureInfo.InvariantCulture, "Enter one of {0}/{1}/{2}/{3}/{4}/{5} (leave blank for default): ", nameof(LogLevel.Trace), nameof(LogLevel.Debug), nameof(LogLevel.Information), nameof(LogLevel.Warning), nameof(LogLevel.Error), nameof(LogLevel.Critical)), false, cancellationToken).ConfigureAwait(false);

                        var responseString = await console.ReadLineAsync(false, cancellationToken).ConfigureAwait(false);

                        if (String.IsNullOrWhiteSpace(responseString))
                        {
                            return(null);
                        }
                        if (Enum.TryParse <LogLevel>(responseString, out var logLevel) && logLevel != LogLevel.None)
                        {
                            return(logLevel);
                        }
                        await console.WriteAsync("Invalid log level!", true, cancellationToken).ConfigureAwait(false);
                    } while (true);
                }

                fileLoggingConfiguration.LogLevel = await PromptLogLevel(String.Format(CultureInfo.InvariantCulture, "Enter the level limit for normal logs (default {0}).", fileLoggingConfiguration.LogLevel)).ConfigureAwait(false) ?? fileLoggingConfiguration.LogLevel;

                fileLoggingConfiguration.MicrosoftLogLevel = await PromptLogLevel(String.Format(CultureInfo.InvariantCulture, "Enter the level limit for Microsoft logs (VERY verbose, default {0}).", fileLoggingConfiguration.MicrosoftLogLevel)).ConfigureAwait(false) ?? fileLoggingConfiguration.MicrosoftLogLevel;
            }
            return(fileLoggingConfiguration);
        }
        public async Task <IActionResult> Create([FromBody] Api.Models.Instance model, CancellationToken cancellationToken)
        {
            if (model == null)
            {
                throw new ArgumentNullException(nameof(model));
            }

            if (String.IsNullOrWhiteSpace(model.Name))
            {
                return(BadRequest(new ErrorMessage {
                    Message = "name must not be empty!"
                }));
            }

            if (model.Path == null)
            {
                return(BadRequest(new ErrorMessage {
                    Message = "path must not be empty!"
                }));
            }

            NormalizeModelPath(model, out var rawPath);

            var localPath = ioManager.ResolvePath(".");

            NormalizeModelPath(new Api.Models.Instance
            {
                Path = localPath
            }, out var normalizedLocalPath);

            if (rawPath.StartsWith(normalizedLocalPath, StringComparison.Ordinal))
            {
                bool sameLength       = rawPath.Length == normalizedLocalPath.Length;
                char dirSeparatorChar = rawPath.ToCharArray()[normalizedLocalPath.Length];
                if (sameLength ||
                    dirSeparatorChar == Path.DirectorySeparatorChar ||
                    dirSeparatorChar == Path.AltDirectorySeparatorChar)
                {
                    return(Conflict(new ErrorMessage {
                        Message = "Instances cannot be created in the installation directory!"
                    }));
                }
            }

            var  dirExistsTask = ioManager.DirectoryExists(model.Path, cancellationToken);
            bool attached      = false;

            if (await ioManager.FileExists(model.Path, cancellationToken).ConfigureAwait(false) || await dirExistsTask.ConfigureAwait(false))
            {
                if (!await ioManager.FileExists(ioManager.ConcatPath(model.Path, InstanceAttachFileName), cancellationToken).ConfigureAwait(false))
                {
                    return(Conflict(new ErrorMessage {
                        Message = "Path not empty!"
                    }));
                }
                else
                {
                    attached = true;
                }
            }

            var newInstance = new Models.Instance
            {
                ConfigurationType   = model.ConfigurationType ?? ConfigurationType.Disallowed,
                DreamDaemonSettings = new DreamDaemonSettings
                {
                    AllowWebClient = false,
                    AutoStart      = false,
                    PrimaryPort    = 1337,
                    SecondaryPort  = 1338,
                    SecurityLevel  = DreamDaemonSecurity.Safe,
                    SoftRestart    = false,
                    SoftShutdown   = false,
                    StartupTimeout = 20
                },
                DreamMakerSettings = new DreamMakerSettings
                {
                    ApiValidationPort          = 1339,
                    ApiValidationSecurityLevel = DreamDaemonSecurity.Safe
                },
                Name               = model.Name,
                Online             = false,
                Path               = model.Path,
                AutoUpdateInterval = model.AutoUpdateInterval ?? 0,
                RepositorySettings = new RepositorySettings
                {
                    CommitterEmail            = "*****@*****.**",
                    CommitterName             = application.VersionPrefix,
                    PushTestMergeCommits      = false,
                    ShowTestMergeCommitters   = false,
                    AutoUpdatesKeepTestMerges = false,
                    AutoUpdatesSynchronize    = false,
                    PostTestMergeComment      = false
                },
                InstanceUsers = new List <Models.InstanceUser>                // give this user full privileges on the instance
                {
                    InstanceAdminUser()
                }
            };

            DatabaseContext.Instances.Add(newInstance);
            try
            {
                await DatabaseContext.Save(cancellationToken).ConfigureAwait(false);

                try
                {
                    // actually reserve it now
                    await ioManager.CreateDirectory(rawPath, cancellationToken).ConfigureAwait(false);

                    await ioManager.DeleteFile(ioManager.ConcatPath(rawPath, InstanceAttachFileName), cancellationToken).ConfigureAwait(false);
                }
                catch
                {
                    // oh shit delete the model
                    DatabaseContext.Instances.Remove(newInstance);

                    await DatabaseContext.Save(default).ConfigureAwait(false);
Example #22
0
        /// <inheritdoc />
                #pragma warning disable CA1506 // TODO: Decomplexify
        public async Task <IDmbProvider> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken)
        {
            if (compileJob == null)
            {
                throw new ArgumentNullException(nameof(compileJob));
            }

            // ensure we have the entire metadata tree
            logger.LogTrace("Loading compile job {0}...", compileJob.Id);
            await databaseContextFactory.UseContext(
                async db => compileJob = await db
                .CompileJobs
                .AsQueryable()
                .Where(x => x.Id == compileJob.Id)
                .Include(x => x.Job)
                .ThenInclude(x => x.StartedBy)
                .Include(x => x.RevisionInformation)
                .ThenInclude(x => x.PrimaryTestMerge)
                .ThenInclude(x => x.MergedBy)
                .Include(x => x.RevisionInformation)
                .ThenInclude(x => x.ActiveTestMerges)
                .ThenInclude(x => x.TestMerge)
                .ThenInclude(x => x.MergedBy)
                .FirstAsync(cancellationToken)
                .ConfigureAwait(false))
            .ConfigureAwait(false);                     // can't wait to see that query

            if (!compileJob.Job.StoppedAt.HasValue)
            {
                // This happens when we're told to load the compile job that is currently finished up
                // It constitutes an API violation if it's returned by the DreamDaemonController so just set it here
                // Bit of a hack, but it works out to be nearly if not the same value that's put in the DB
                logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{0}...", compileJob.Job.Id);
                compileJob.Job.StoppedAt = DateTimeOffset.UtcNow;
            }

            var providerSubmitted = false;

            void CleanupAction()
            {
                if (providerSubmitted)
                {
                    CleanJob(compileJob);
                }
            }

            var newProvider = new DmbProvider(compileJob, ioManager, CleanupAction);

            try
            {
                const string LegacyADirectoryName = "A";
                const string LegacyBDirectoryName = "B";

                var dmbExistsAtRoot = await ioManager.FileExists(
                    ioManager.ConcatPath(
                        newProvider.Directory,
                        newProvider.DmbName),
                    cancellationToken)
                                      .ConfigureAwait(false);

                if (!dmbExistsAtRoot)
                {
                    logger.LogTrace("Didn't find .dmb at game directory root, checking A/B dirs...");
                    var primaryCheckTask = ioManager.FileExists(
                        ioManager.ConcatPath(
                            newProvider.Directory,
                            LegacyADirectoryName,
                            newProvider.DmbName),
                        cancellationToken);
                    var secondaryCheckTask = ioManager.FileExists(
                        ioManager.ConcatPath(
                            newProvider.Directory,
                            LegacyBDirectoryName,
                            newProvider.DmbName),
                        cancellationToken);

                    if (!(await primaryCheckTask.ConfigureAwait(false) && await secondaryCheckTask.ConfigureAwait(false)))
                    {
                        logger.LogWarning("Error loading compile job, .dmb missing!");
                        return(null);                        // omae wa mou shinderu
                    }

                    // rebuild the provider because it's using the legacy style directories
                    // Don't dispose it
                    logger.LogDebug("Creating legacy two folder .dmb provider targeting {0} directory...", LegacyADirectoryName);
                    newProvider = new DmbProvider(compileJob, ioManager, CleanupAction, Path.DirectorySeparatorChar + LegacyADirectoryName);
                }

                lock (jobLockCounts)
                {
                    if (!jobLockCounts.TryGetValue(compileJob.Id, out int value))
                    {
                        value = 1;
                        jobLockCounts.Add(compileJob.Id, 1);
                    }
                    else
                    {
                        jobLockCounts[compileJob.Id] = ++value;
                    }

                    providerSubmitted = true;

                    logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value);
                    return(newProvider);
                }
            }
            finally
            {
                if (!providerSubmitted)
                {
                    newProvider.Dispose();
                }
            }
        }
Example #23
0
        /// <summary>
        /// Generate map diffs for a given <paramref name="pullRequest"/>
        /// </summary>
        /// <param name="pullRequest">The <see cref="PullRequest"/></param>
        /// <param name="checkRunId">The <see cref="CheckRun.Id"/></param>
        /// <param name="changedDmms">Paths to changed .dmm files</param>
        /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
        /// <returns>A <see cref="Task"/> representing the running operation</returns>
        async Task GenerateDiffs(PullRequest pullRequest, long checkRunId, IReadOnlyList <string> changedDmms, CancellationToken cancellationToken)
        {
            using (logger.BeginScope("Generating {0} diffs for pull request #{1} in {2}/{3}", changedDmms.Count, pullRequest.Number, pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name))
            {
                const string OldMapExtension = ".old_map_diff_bot";

                var gitHubManager = serviceProvider.GetRequiredService <IGitHubManager>();

                Task generatingCommentTask;
                List <Task <RenderResult> > afterRenderings, beforeRenderings;

                var workingDir = ioManager.ConcatPath(pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name, pullRequest.Number.ToString(CultureInfo.InvariantCulture));
                logger.LogTrace("Setting workdir to {0}", workingDir);
                IIOManager currentIOManager = new ResolvingIOManager(ioManager, workingDir);
                string     repoPath;

                int  lastProgress       = -1;
                Task lastProgressUpdate = Task.CompletedTask;
                async Task OnCloneProgress(int progress)
                {
                    lock (gitHubManager)
                    {
                        if (lastProgress >= progress)
                        {
                            return;
                        }
                        if (lastProgress == -1)
                        {
                            logger.LogInformation("Waiting on repository to finish cloning...");
                        }
                        lastProgress = progress;
                    }
                    await lastProgressUpdate.ConfigureAwait(false);

                    await gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate
                    {
                        Status = CheckStatus.InProgress,
                        Output = new CheckRunOutput(stringLocalizer["Cloning Repository"], stringLocalizer["Clone Progress: {0}%", progress], null, null, null),
                    }, cancellationToken).ConfigureAwait(false);
                };
                Task CreateBlockedComment()
                {
                    logger.LogInformation("Waiting for another diff generation on {0}/{1} to complete...", pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name);
                    return(gitHubManager.CreateSingletonComment(pullRequest, stringLocalizer["Waiting for another operation on this repository to complete..."], cancellationToken));
                };

                logger.LogTrace("Locking repository...");
                using (var repo = await repositoryManager.GetRepository(pullRequest.Base.Repository, OnCloneProgress, CreateBlockedComment, cancellationToken).ConfigureAwait(false))
                {
                    logger.LogTrace("Repository ready");
                    generatingCommentTask = gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate
                    {
                        Status = CheckStatus.InProgress,
                        Output = new CheckRunOutput(stringLocalizer["Generating Diffs"], stringLocalizer["Aww geez rick, I should eventually put some progress message here"], null, null, null),
                    }, cancellationToken);
                    //prep the outputDirectory
                    async Task DirectoryPrep()
                    {
                        logger.LogTrace("Cleaning workdir...");
                        await currentIOManager.DeleteDirectory(".", cancellationToken).ConfigureAwait(false);

                        await currentIOManager.CreateDirectory(".", cancellationToken).ConfigureAwait(false);

                        logger.LogTrace("Workdir cleaned");
                    };

                    var dirPrepTask = DirectoryPrep();
                    //get the dme to use
                    var dmeToUseTask = serviceProvider.GetRequiredService <IDatabaseContext>().InstallationRepositories.Where(x => x.Id == pullRequest.Base.Repository.Id).Select(x => x.TargetDme).ToAsyncEnumerable().FirstOrDefault(cancellationToken);

                    var oldMapPaths = new List <string>()
                    {
                        Capacity = changedDmms.Count
                    };
                    try
                    {
                        //fetch base commit if necessary and check it out, fetch pull request
                        if (!await repo.ContainsCommit(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false))
                        {
                            logger.LogTrace("Base commit not found, running fetch...");
                            await repo.Fetch(cancellationToken).ConfigureAwait(false);
                        }
                        logger.LogTrace("Moving HEAD to pull request base...");
                        await repo.Checkout(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false);

                        //but since we don't need this right await don't await it yet
                        var pullRequestFetchTask = repo.FetchPullRequest(pullRequest.Number, cancellationToken);
                        try
                        {
                            //first copy all modified maps to the same location with the .old_map_diff_bot extension
                            async Task <string> CacheMap(string mapPath)
                            {
                                var originalPath = currentIOManager.ConcatPath(repoPath, mapPath);

                                if (await currentIOManager.FileExists(originalPath, cancellationToken).ConfigureAwait(false))
                                {
                                    logger.LogTrace("Creating old map cache of {0}", mapPath);
                                    var oldMapPath = String.Format(CultureInfo.InvariantCulture, "{0}{1}", originalPath, OldMapExtension);
                                    await currentIOManager.CopyFile(originalPath, oldMapPath, cancellationToken).ConfigureAwait(false);

                                    return(oldMapPath);
                                }
                                return(null);
                            };

                            repoPath = repo.Path;

                            var tasks = changedDmms.Select(x => CacheMap(x)).ToList();
                            await Task.WhenAll(tasks).ConfigureAwait(false);

                            oldMapPaths.AddRange(tasks.Select(x => x.Result));
                        }
                        finally
                        {
                            logger.LogTrace("Waiting for pull request commits to be available...");
                            await pullRequestFetchTask.ConfigureAwait(false);
                        }

                        logger.LogTrace("Creating and moving HEAD to pull request merge commit...");
                        //generate the merge commit ourselves since we can't get it from GitHub because itll return an outdated one
                        await repo.Merge(pullRequest.Head.Sha, cancellationToken).ConfigureAwait(false);
                    }
                    finally
                    {
                        logger.LogTrace("Waiting for configured project dme...");
                        await dmeToUseTask.ConfigureAwait(false);
                    }

                    //create empty array of map regions
                    var mapRegions = Enumerable.Repeat <MapRegion>(null, changedDmms.Count).ToList();
                    var dmeToUse   = dmeToUseTask.Result;

                    var generator       = generatorFactory.CreateGenerator(dmeToUse, new ResolvingIOManager(ioManager, repoPath));
                    var outputDirectory = currentIOManager.ResolvePath(".");
                    logger.LogTrace("Full workdir path: {0}", outputDirectory);
                    //Generate MapRegions for modified maps and render all new maps
                    async Task <RenderResult> DiffAndRenderNewMap(int I)
                    {
                        await dirPrepTask.ConfigureAwait(false);

                        var originalPath = currentIOManager.ConcatPath(repoPath, changedDmms[I]);

                        if (!await currentIOManager.FileExists(originalPath, cancellationToken).ConfigureAwait(false))
                        {
                            logger.LogTrace("No new map for path {0} exists, skipping region detection and after render", changedDmms[I]);
                            return(new RenderResult {
                                InputPath = changedDmms[I], ToolOutput = stringLocalizer["Map missing!"]
                            });
                        }
                        ToolResult result = null;

                        if (oldMapPaths[I] != null)
                        {
                            logger.LogTrace("Getting diff region for {0}...", changedDmms[I]);
                            result = await generator.GetDifferences(oldMapPaths[I], originalPath, cancellationToken).ConfigureAwait(false);

                            var region = result.MapRegion;
                            logger.LogTrace("Diff region for {0}: {1}", changedDmms[I], region);
                            if (region != null)
                            {
                                var       xdiam             = region.MaxX - region.MinX;
                                var       ydiam             = region.MaxY - region.MinY;
                                const int minDiffDimensions = 5 - 1;
                                if (xdiam < minDiffDimensions || ydiam < minDiffDimensions)
                                {
                                    //need to expand
                                    var fullResult = await generator.GetMapSize(originalPath, cancellationToken).ConfigureAwait(false);

                                    var fullRegion = fullResult.MapRegion;
                                    if (fullRegion == null)
                                    {
                                        //give up
                                        region = null;
                                    }
                                    else
                                    {
                                        bool increaseMax = true;
                                        if (xdiam < minDiffDimensions && ((fullRegion.MaxX - fullRegion.MinX) >= minDiffDimensions))
                                        {
                                            while ((region.MaxX - region.MinX) < minDiffDimensions)
                                            {
                                                if (increaseMax)
                                                {
                                                    region.MaxX = (short)Math.Min(region.MaxX + 1, fullRegion.MaxX);
                                                }
                                                else
                                                {
                                                    region.MinX = (short)Math.Max(region.MinX - 1, 1);
                                                }
                                                increaseMax = !increaseMax;
                                            }
                                        }
                                        if (ydiam < minDiffDimensions && ((fullRegion.MaxY - fullRegion.MinY) >= minDiffDimensions))
                                        {
                                            while ((region.MaxY - region.MinY) < minDiffDimensions)
                                            {
                                                if (increaseMax)
                                                {
                                                    region.MaxY = (short)Math.Min(region.MaxY + 1, fullRegion.MaxY);
                                                }
                                                else
                                                {
                                                    region.MinY = (short)Math.Max(region.MinY - 1, 1);
                                                }
                                                increaseMax = !increaseMax;
                                            }
                                        }
                                    }
                                    logger.LogTrace("Region for {0} expanded to {1}", changedDmms[I], region);
                                }
                                mapRegions[I] = region;
                            }
                        }
                        else
                        {
                            logger.LogTrace("Skipping region detection for {0} due to old map not existing", changedDmms[I]);
                        }
                        logger.LogTrace("Performing after rendering for {0}...", changedDmms[I]);
                        var renderResult = await generator.RenderMap(originalPath, mapRegions[I], outputDirectory, "after", cancellationToken).ConfigureAwait(false);

                        logger.LogTrace("After rendering for {0} complete! Result path: {1}, Output: {2}", changedDmms[I], renderResult.OutputPath, renderResult.ToolOutput);
                        if (result != null)
                        {
                            renderResult.ToolOutput = String.Format(CultureInfo.InvariantCulture, "Differences task:{0}{1}{0}Render task:{0}{2}", Environment.NewLine, result.ToolOutput, renderResult.ToolOutput);
                        }
                        return(renderResult);
                    };

                    logger.LogTrace("Running iterations of DiffAndRenderNewMap...");
                    //finish up before we go back to the base branch
                    afterRenderings = Enumerable.Range(0, changedDmms.Count).Select(I => DiffAndRenderNewMap(I)).ToList();
                    try
                    {
                        await Task.WhenAll(afterRenderings).ConfigureAwait(false);
                    }
                    catch (Exception e)
                    {
                        logger.LogDebug(e, "After renderings produced exception!");
                        //at this point everything is done but some have failed
                        //we'll handle it later
                    }

                    logger.LogTrace("Moving HEAD back to pull request base...");
                    await repo.Checkout(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false);

                    Task <RenderResult> RenderOldMap(int i)
                    {
                        var oldPath = oldMapPaths[i];

                        if (oldMapPaths != null)
                        {
                            logger.LogTrace("Performing before rendering for {0}...", changedDmms[i]);
                            return(generator.RenderMap(oldPath, mapRegions[i], outputDirectory, "before", cancellationToken));
                        }
                        return(Task.FromResult(new RenderResult {
                            InputPath = changedDmms[i], ToolOutput = stringLocalizer["Map missing!"]
                        }));
                    }

                    logger.LogTrace("Running iterations of RenderOldMap...");
                    //finish up rendering
                    beforeRenderings = Enumerable.Range(0, changedDmms.Count).Select(I => RenderOldMap(I)).ToList();
                    try
                    {
                        await Task.WhenAll(beforeRenderings).ConfigureAwait(false);
                    }
                    catch (Exception e)
                    {
                        logger.LogDebug(e, "Before renderings produced exception!");
                        //see above
                    }

                    //done with the repo at this point
                    logger.LogTrace("Renderings complete. Releasing reposiotory");
                }

                //collect results and errors
                async Task <KeyValuePair <MapDiff, MapRegion> > GetResult(int i)
                {
                    var beforeTask = beforeRenderings[i];
                    var afterTask  = afterRenderings[i];

                    var result = new MapDiff
                    {
                        InstallationRepositoryId = pullRequest.Base.Repository.Id,
                        CheckRunId = checkRunId,
                        FileId     = i,
                    };

                    RenderResult GetRenderingResult(Task <RenderResult> task)
                    {
                        if (task.Exception != null)
                        {
                            result.LogMessage = result.LogMessage == null?task.Exception.ToString() : String.Format(CultureInfo.InvariantCulture, "{0}{1}{2}", result.LogMessage, Environment.NewLine, task.Exception);

                            return(null);
                        }
                        return(task.Result);
                    };

                    var r1 = GetRenderingResult(beforeTask);
                    var r2 = GetRenderingResult(afterTask);

                    logger.LogTrace("Results for {0}: Before {1}, After {2}", changedDmms[i], r1?.OutputPath ?? "NONE", r2?.OutputPath ?? "NONE");

                    result.MapPath = changedDmms[i];

                    result.LogMessage = String.Format(CultureInfo.InvariantCulture, "Job {5}:{0}Path: {6}{0}Before:{0}Command Line: {1}{0}Output:{0}{2}{0}Logs:{0}{7}{0}After:{0}Command Line: {3}{0}Output:{0}{4}{0}Logs:{0}{8}{0}Exceptions:{0}{9}{0}", Environment.NewLine, r1?.CommandLine, r1?.OutputPath, r2?.CommandLine, r2?.OutputPath, i + 1, result.MapPath, r1?.ToolOutput, r2?.ToolOutput, result.LogMessage);

                    async Task <byte[]> ReadMapImage(string path)
                    {
                        if (path != null && await currentIOManager.FileExists(path, cancellationToken).ConfigureAwait(false))
                        {
                            var bytes = await currentIOManager.ReadAllBytes(path, cancellationToken).ConfigureAwait(false);

                            await currentIOManager.DeleteFile(path, cancellationToken).ConfigureAwait(false);

                            return(bytes);
                        }
                        return(null);
                    }

                    var readBeforeTask = ReadMapImage(r1?.OutputPath);

                    result.AfterImage = await ReadMapImage(r2?.OutputPath).ConfigureAwait(false);

                    result.BeforeImage = await readBeforeTask.ConfigureAwait(false);

                    return(new KeyValuePair <MapDiff, MapRegion>(result, r2?.MapRegion));
                }

                logger.LogTrace("Waiting for notification comment to POST...");
                await generatingCommentTask.ConfigureAwait(false);

                logger.LogTrace("Collecting results...");
                var results = Enumerable.Range(0, changedDmms.Count).Select(x => GetResult(x)).ToList();
                await Task.WhenAll(results).ConfigureAwait(false);

                var dic = new Dictionary <MapDiff, MapRegion>();
                foreach (var I in results.Select(x => x.Result))
                {
                    dic.Add(I.Key, I.Value);
                }
                await HandleResults(pullRequest, checkRunId, dic, cancellationToken).ConfigureAwait(false);
            }
        }
Example #24
0
 /// <summary>
 /// Construct a <see cref="Repository"/>
 /// </summary>
 /// <param name="gitHubConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the <see cref="GitHubConfiguration"/> to use for determining the <see cref="repositoryObject"/>'s path</param>
 /// <param name="_logger">The <see cref="ILogger"/> to use for setting up the <see cref="Repository"/></param>
 /// <param name="_ioManager">The value of <see cref="ioManager"/></param>
 public Repository(IOptions <GitHubConfiguration> gitHubConfigurationOptions, ILogger <Repository> _logger, IIOManager _ioManager)
 {
     logger = _logger ?? throw new ArgumentNullException(nameof(_logger));
     gitHubConfiguration = gitHubConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(gitHubConfigurationOptions));
     ioManager           = new ResolvingIOManager(_ioManager ?? throw new ArgumentNullException(nameof(_ioManager)), _ioManager.ConcatPath(Application.DataDirectory, RepositoriesDirectory));
     semaphore           = new SemaphoreSlim(1);
 }
Example #25
0
        /// <inheritdoc />
                #pragma warning disable CA1506 // TODO: Decomplexify
        public async Task <IDmbProvider> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken)
        {
            if (compileJob == null)
            {
                throw new ArgumentNullException(nameof(compileJob));
            }

            // ensure we have the entire compile job tree
            logger.LogTrace("Loading compile job {0}...", compileJob.Id);
            await databaseContextFactory.UseContext(
                async db => compileJob = await db
                .CompileJobs
                .AsQueryable()
                .Where(x => x.Id == compileJob.Id)
                .Include(x => x.Job).ThenInclude(x => x.StartedBy)
                .Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge).ThenInclude(x => x.MergedBy)
                .Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy)
                .FirstAsync(cancellationToken)
                .ConfigureAwait(false))
            .ConfigureAwait(false);                     // can't wait to see that query

            if (!compileJob.Job.StoppedAt.HasValue)
            {
                // This happens if we're told to load the compile job that is currently finished up
                // It can constitute an API violation if it's returned by the DreamDaemonController so just set it here
                // Bit of a hack, but it should work out to be the same value
                logger.LogTrace("Setting missing StoppedAt for CompileJob job...");
                compileJob.Job.StoppedAt = DateTimeOffset.Now;
            }

            var providerSubmitted = false;
            var newProvider       = new DmbProvider(compileJob, ioManager, () =>
            {
                if (providerSubmitted)
                {
                    CleanJob(compileJob);
                }
            });

            try
            {
                var primaryCheckTask   = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken);
                var secondaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken);

                if (!(await primaryCheckTask.ConfigureAwait(false) && await secondaryCheckTask.ConfigureAwait(false)))
                {
                    logger.LogWarning("Error loading compile job, .dmb missing!");
                    return(null);                    // omae wa mou shinderu
                }

                lock (jobLockCounts)
                {
                    if (!jobLockCounts.TryGetValue(compileJob.Id, out int value))
                    {
                        value = 1;
                        jobLockCounts.Add(compileJob.Id, 1);
                    }
                    else
                    {
                        jobLockCounts[compileJob.Id] = ++value;
                    }

                    logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value);

                    providerSubmitted = true;
                    return(newProvider);
                }
            }
            finally
            {
                if (!providerSubmitted)
                {
                    newProvider.Dispose();
                }
            }
        }
Example #26
0
        /// <inheritdoc />
        public Task Initialize(CancellationToken cancellationToken)
        {
            return(Task.Factory.StartNew(async() =>
            {
                using (logger.BeginScope("Initializing repository..."))
                {
                    var repoPath = ioManager.ResolvePath(ioManager.ConcatPath(gitHubConfiguration.RepoOwner, gitHubConfiguration.RepoName));

                    logger.LogTrace("Repo path evaluated to be: {0}", repoPath);

                    try
                    {
                        logger.LogTrace("Creating repository object.");
                        cancellationToken.ThrowIfCancellationRequested();
                        repositoryObject = new LibGit2Sharp.Repository(repoPath);

                        repositoryObject.RemoveUntrackedFiles();

                        cancellationToken.ThrowIfCancellationRequested();

                        repositoryObject.RetrieveStatus();
                    }
                    catch (OperationCanceledException e)
                    {
                        logger.LogDebug(e, "Repository setup cancelled!");
                        repositoryObject?.Dispose();
                        throw;
                    }
                    catch (Exception e)
                    {
                        cancellationToken.ThrowIfCancellationRequested();
                        using (logger.BeginScope("Repository fallback initializing..."))
                        {
                            repositoryObject?.Dispose();
                            try
                            {
                                logger.LogTrace("Checking repository directory exists.");
                                if (await ioManager.DirectoryExists(repoPath, cancellationToken).ConfigureAwait(false))
                                {
                                    logger.LogWarning(e, "Failed to load repository! Deleting and cloning...");
                                    await ioManager.DeleteDirectory(repoPath, cancellationToken).ConfigureAwait(false);
                                }
                                else
                                {
                                    logger.LogInformation(e, "Cloning repository...");
                                }

                                LibGit2Sharp.Repository.Clone(String.Format(CultureInfo.InvariantCulture, "https://github.com/{0}/{1}", gitHubConfiguration.RepoOwner, gitHubConfiguration.RepoName), repoPath, new CloneOptions
                                {
                                    Checkout = false,
                                    RecurseSubmodules = true,
                                    OnProgress = (a) => !cancellationToken.IsCancellationRequested,
                                    OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
                                    OnTransferProgress = (a) => !cancellationToken.IsCancellationRequested
                                });

                                logger.LogInformation("Repo clone completed.");

                                repositoryObject = new LibGit2Sharp.Repository(repoPath);
                            }
                            catch (UserCancelledException e2)
                            {
                                logger.LogDebug(e2, "Repository setup cancelled!");
                                cancellationToken.ThrowIfCancellationRequested();
                            }
                            catch (Exception e2)
                            {
                                logger.LogCritical(e2, "Unable to clone repository!");
                                throw;
                            }
                        }
                    }
                }
            }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current));
        }
        /// <summary>
        /// Configure dependency injected services
        /// </summary>
        /// <param name="services">The <see cref="IServiceCollection"/> to configure</param>
        public void ConfigureServices(IServiceCollection services)
        {
            if (services == null)
            {
                throw new ArgumentNullException(nameof(services));
            }

            // needful
            services.AddSingleton <IApplication>(this);

            // configure configuration
            services.UseStandardConfig <UpdatesConfiguration>(configuration);
            services.UseStandardConfig <DatabaseConfiguration>(configuration);
            services.UseStandardConfig <GeneralConfiguration>(configuration);
            services.UseStandardConfig <FileLoggingConfiguration>(configuration);
            services.UseStandardConfig <ControlPanelConfiguration>(configuration);

            // enable options which give us config reloading
            services.AddOptions();

            // this is needful for the setup wizard
            services.AddLogging();

            // other stuff needed for for setup wizard and configuration
            services.AddSingleton <IConsole, IO.Console>();
            services.AddSingleton <IDatabaseConnectionFactory, DatabaseConnectionFactory>();
            services.AddSingleton <ISetupWizard, SetupWizard>();
            services.AddSingleton <IPlatformIdentifier, PlatformIdentifier>();
            services.AddSingleton <IAsyncDelayer, AsyncDelayer>();

            GeneralConfiguration      generalConfiguration;
            DatabaseConfiguration     databaseConfiguration;
            FileLoggingConfiguration  fileLoggingConfiguration;
            ControlPanelConfiguration controlPanelConfiguration;
            IPlatformIdentifier       platformIdentifier;

            // temporarily build the service provider in it's current state
            // do it here so we can run the setup wizard if necessary
            // also allows us to get some options and other services we need for continued configuration
            using (var provider = services.BuildServiceProvider())
            {
                // run the wizard if necessary
                var setupWizard         = provider.GetRequiredService <ISetupWizard>();
                var applicationLifetime = provider.GetRequiredService <Microsoft.AspNetCore.Hosting.IApplicationLifetime>();
                var setupWizardRan      = setupWizard.CheckRunWizard(applicationLifetime.ApplicationStopping).GetAwaiter().GetResult();

                // load the configuration options we need
                var generalOptions = provider.GetRequiredService <IOptions <GeneralConfiguration> >();
                generalConfiguration = generalOptions.Value;

                // unless this is set, in which case, we leave
                if (setupWizardRan && generalConfiguration.SetupWizardMode == SetupWizardMode.Only)
                {
                    throw new OperationCanceledException("Exiting due to SetupWizardMode configuration!");                     // we don't inject a logger in the constuctor to log this because it's not yet configured
                }
                var dbOptions = provider.GetRequiredService <IOptions <DatabaseConfiguration> >();
                databaseConfiguration = dbOptions.Value;

                var loggingOptions = provider.GetRequiredService <IOptions <FileLoggingConfiguration> >();
                fileLoggingConfiguration = loggingOptions.Value;

                var controlPanelOptions = provider.GetRequiredService <IOptions <ControlPanelConfiguration> >();
                controlPanelConfiguration = controlPanelOptions.Value;

                platformIdentifier = provider.GetRequiredService <IPlatformIdentifier>();
            }

            // setup file logging via serilog
            if (!fileLoggingConfiguration.Disable)
            {
                services.AddLogging(builder =>
                {
                    // common app data is C:/ProgramData on windows, else /usr/shar
                    var logPath = !String.IsNullOrEmpty(fileLoggingConfiguration.Directory) ? fileLoggingConfiguration.Directory : ioManager.ConcatPath(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), VersionPrefix, "Logs");

                    logPath = ioManager.ConcatPath(logPath, "tgs-{Date}.log");

                    LogEventLevel?ConvertLogLevel(LogLevel logLevel)
                    {
                        switch (logLevel)
                        {
                        case LogLevel.Critical:
                            return(LogEventLevel.Fatal);

                        case LogLevel.Debug:
                            return(LogEventLevel.Debug);

                        case LogLevel.Error:
                            return(LogEventLevel.Error);

                        case LogLevel.Information:
                            return(LogEventLevel.Information);

                        case LogLevel.Trace:
                            return(LogEventLevel.Verbose);

                        case LogLevel.Warning:
                            return(LogEventLevel.Warning);

                        case LogLevel.None:
                            return(null);

                        default:
                            throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Invalid log level {0}", logLevel));
                        }
                    }

                    var logEventLevel       = ConvertLogLevel(fileLoggingConfiguration.LogLevel);
                    var microsoftEventLevel = ConvertLogLevel(fileLoggingConfiguration.MicrosoftLogLevel);

                    var formatter = new MessageTemplateTextFormatter("{Timestamp:o} {RequestId,13} [{Level:u3}] {SourceContext:l}: {Message} ({EventId:x8}){NewLine}{Exception}", null);

                    var configuration = new LoggerConfiguration()
                                        .Enrich.FromLogContext()
                                        .WriteTo.Async(w => w.RollingFile(formatter, logPath, shared: true, flushToDiskInterval: TimeSpan.FromSeconds(2)));

                    if (logEventLevel.HasValue)
                    {
                        configuration.MinimumLevel.Is(logEventLevel.Value);
                    }

                    if (microsoftEventLevel.HasValue)
                    {
                        configuration.MinimumLevel.Override("Microsoft", microsoftEventLevel.Value);
                    }

                    builder.AddSerilog(configuration.CreateLogger(), true);
                });
            }

            // configure bearer token validation
            services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme).AddJwtBearer(jwtBearerOptions =>
            {
                // this line isn't actually run until the first request is made
                // at that point tokenFactory will be populated
                jwtBearerOptions.TokenValidationParameters = tokenFactory.ValidationParameters;
                jwtBearerOptions.Events = new JwtBearerEvents
                {
                    // Application is our composition root so this monstrosity of a line is okay
                    OnTokenValidated = ctx => ctx.HttpContext.RequestServices.GetRequiredService <IClaimsInjector>().InjectClaimsIntoContext(ctx, ctx.HttpContext.RequestAborted)
                };
            });

            // f*****g prevents converting 'sub' to M$ bs
            // can't be done in the above lambda, that's too late
            JwtSecurityTokenHandler.DefaultInboundClaimTypeMap.Clear();

            // add mvc, configure the json serializer settings
            services
            .AddMvc(options =>
            {
                var dataAnnotationValidator = options.ModelValidatorProviders.Single(validator => validator.GetType().Name == "DataAnnotationsModelValidatorProvider");
                options.ModelValidatorProviders.Remove(dataAnnotationValidator);
            })
            .SetCompatibilityVersion(CompatibilityVersion.Version_2_1)
            .AddJsonOptions(options =>
            {
                options.AllowInputFormatterExceptionMessages      = true;
                options.SerializerSettings.NullValueHandling      = NullValueHandling.Ignore;
                options.SerializerSettings.CheckAdditionalContent = true;
                options.SerializerSettings.MissingMemberHandling  = MissingMemberHandling.Error;
                options.SerializerSettings.ReferenceLoopHandling  = ReferenceLoopHandling.Ignore;
                options.SerializerSettings.Converters             = new[] { new VersionConverter() };
            });

            if (hostingEnvironment.IsDevelopment())
            {
                string GetDocumentationFilePath(string assemblyLocation) => ioManager.ConcatPath(ioManager.GetDirectoryName(assemblyLocation), String.Concat(ioManager.GetFileNameWithoutExtension(assemblyLocation), ".xml"));

                var assemblyDocumentationPath = GetDocumentationFilePath(assemblyInformationProvider.Path);
                var apiDocumentationPath      = GetDocumentationFilePath(typeof(ApiHeaders).Assembly.Location);
                services.AddSwaggerGen(genOptions => SwaggerConfiguration.Configure(genOptions, assemblyDocumentationPath, apiDocumentationPath));
            }

            // enable browser detection
            services.AddDetectionCore().AddBrowser();

            // CORS conditionally enabled later
            services.AddCors();

            void AddTypedContext <TContext>() where TContext : DatabaseContext <TContext>
            {
                services.AddDbContext <TContext>(builder =>
                {
                    if (hostingEnvironment.IsDevelopment())
                    {
                        builder.EnableSensitiveDataLogging();
                    }
                });
                services.AddScoped <IDatabaseContext>(x => x.GetRequiredService <TContext>());
            }

            // add the correct database context type
            var dbType = databaseConfiguration.DatabaseType;

            switch (dbType)
            {
            case DatabaseType.MySql:
            case DatabaseType.MariaDB:
                AddTypedContext <MySqlDatabaseContext>();
                break;

            case DatabaseType.SqlServer:
                AddTypedContext <SqlServerDatabaseContext>();
                break;

            default:
                throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Invalid {0}: {1}!", nameof(DatabaseType), dbType));
            }

            // configure other database services
            services.AddSingleton <IDatabaseContextFactory, DatabaseContextFactory>();
            services.AddSingleton <IDatabaseSeeder, DatabaseSeeder>();

            // configure security services
            services.AddScoped <IAuthenticationContextFactory, AuthenticationContextFactory>();
            services.AddScoped <IClaimsInjector, ClaimsInjector>();
            services.AddSingleton <IIdentityCache, IdentityCache>();
            services.AddSingleton <ICryptographySuite, CryptographySuite>();
            services.AddSingleton <ITokenFactory, TokenFactory>();
            services.AddSingleton <IPasswordHasher <Models.User>, PasswordHasher <Models.User> >();

            // configure platform specific services
            if (platformIdentifier.IsWindows)
            {
                if (generalConfiguration.UseBasicWatchdogOnWindows)
                {
                    services.AddSingleton <IWatchdogFactory, WatchdogFactory>();
                }
                else
                {
                    services.AddSingleton <IWatchdogFactory, WindowsWatchdogFactory>();
                }

                services.AddSingleton <ISystemIdentityFactory, WindowsSystemIdentityFactory>();
                services.AddSingleton <ISymlinkFactory, WindowsSymlinkFactory>();
                services.AddSingleton <IByondInstaller, WindowsByondInstaller>();
                services.AddSingleton <IPostWriteHandler, WindowsPostWriteHandler>();

                services.AddSingleton <WindowsNetworkPromptReaper>();
                services.AddSingleton <INetworkPromptReaper>(x => x.GetRequiredService <WindowsNetworkPromptReaper>());
                services.AddSingleton <IHostedService>(x => x.GetRequiredService <WindowsNetworkPromptReaper>());
            }
            else
            {
                services.AddSingleton <IWatchdogFactory, WatchdogFactory>();
                services.AddSingleton <ISystemIdentityFactory, PosixSystemIdentityFactory>();
                services.AddSingleton <ISymlinkFactory, PosixSymlinkFactory>();
                services.AddSingleton <IByondInstaller, PosixByondInstaller>();
                services.AddSingleton <IPostWriteHandler, PosixPostWriteHandler>();
                services.AddSingleton <INetworkPromptReaper, PosixNetworkPromptReaper>();
            }

            // configure misc services
            services.AddSingleton <ISynchronousIOManager, SynchronousIOManager>();
            services.AddSingleton <IGitHubClientFactory, GitHubClientFactory>();
            services.AddSingleton <IProcessExecutor, ProcessExecutor>();
            services.AddSingleton <IByondTopicSender>(new ByondTopicSender
            {
                ReceiveTimeout = generalConfiguration.ByondTopicTimeout,
                SendTimeout    = generalConfiguration.ByondTopicTimeout
            });

            // configure component services
            services.AddSingleton <ICredentialsProvider, CredentialsProvider>();
            services.AddSingleton <IProviderFactory, ProviderFactory>();
            services.AddSingleton <IChatFactory, ChatFactory>();
            services.AddSingleton <IInstanceFactory, InstanceFactory>();

            // configure root services
            services.AddSingleton <InstanceManager>();
            services.AddSingleton <IInstanceManager>(x => x.GetRequiredService <InstanceManager>());
            services.AddSingleton <IHostedService>(x => x.GetRequiredService <InstanceManager>());

            services.AddSingleton <IJobManager, JobManager>();
        }
Example #28
0
        public async Task <IActionResult> Create([FromBody] Api.Models.Instance model, CancellationToken cancellationToken)
        {
            if (model == null)
            {
                throw new ArgumentNullException(nameof(model));
            }

            if (String.IsNullOrWhiteSpace(model.Name))
            {
                return(BadRequest(new ErrorMessage(ErrorCode.InstanceWhitespaceName)));
            }

            var unNormalizedPath   = model.Path;
            var targetInstancePath = NormalizePath(unNormalizedPath);

            model.Path = targetInstancePath;

            var installationDirectoryPath = NormalizePath(DefaultIOManager.CurrentDirectory);

            bool InstanceIsChildOf(string otherPath)
            {
                if (!targetInstancePath.StartsWith(otherPath, StringComparison.Ordinal))
                {
                    return(false);
                }

                bool sameLength       = targetInstancePath.Length == otherPath.Length;
                char dirSeparatorChar = targetInstancePath.ToCharArray()[Math.Min(otherPath.Length, targetInstancePath.Length - 1)];

                return(sameLength ||
                       dirSeparatorChar == Path.DirectorySeparatorChar ||
                       dirSeparatorChar == Path.AltDirectorySeparatorChar);
            }

            if (InstanceIsChildOf(installationDirectoryPath))
            {
                return(Conflict(new ErrorMessage(ErrorCode.InstanceAtConflictingPath)));
            }

            // Validate it's not a child of any other instance
            IActionResult earlyOut = null;
            ulong         countOfOtherInstances = 0;

            using (var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
            {
                var newCancellationToken = cts.Token;
                try
                {
                    await DatabaseContext
                    .Instances
                    .AsQueryable()
                    .Select(x => new Models.Instance
                    {
                        Path = x.Path
                    })
                    .ForEachAsync(
                        otherInstance =>
                    {
                        if (++countOfOtherInstances >= generalConfiguration.InstanceLimit)
                        {
                            earlyOut ??= Conflict(new ErrorMessage(ErrorCode.InstanceLimitReached));
                        }
                        else if (InstanceIsChildOf(otherInstance.Path))
                        {
                            earlyOut ??= Conflict(new ErrorMessage(ErrorCode.InstanceAtConflictingPath));
                        }

                        if (earlyOut != null && !newCancellationToken.IsCancellationRequested)
                        {
                            cts.Cancel();
                        }
                    },
                        newCancellationToken)
                    .ConfigureAwait(false);
                }
                catch (OperationCanceledException)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                }
            }

            if (earlyOut != null)
            {
                return(earlyOut);
            }

            // Last test, ensure it's in the list of valid paths
            if (!(generalConfiguration.ValidInstancePaths?
                  .Select(path => NormalizePath(path))
                  .Any(path => InstanceIsChildOf(path)) ?? true))
            {
                return(BadRequest(new ErrorMessage(ErrorCode.InstanceNotAtWhitelistedPath)));
            }

            async Task <bool> DirExistsAndIsNotEmpty()
            {
                if (!await ioManager.DirectoryExists(model.Path, cancellationToken).ConfigureAwait(false))
                {
                    return(false);
                }

                var filesTask = ioManager.GetFiles(model.Path, cancellationToken);
                var dirsTask  = ioManager.GetDirectories(model.Path, cancellationToken);

                var files = await filesTask.ConfigureAwait(false);

                var dirs = await dirsTask.ConfigureAwait(false);

                return(files.Concat(dirs).Any());
            }

            var  dirExistsTask = DirExistsAndIsNotEmpty();
            bool attached      = false;

            if (await ioManager.FileExists(model.Path, cancellationToken).ConfigureAwait(false) || await dirExistsTask.ConfigureAwait(false))
            {
                if (!await ioManager.FileExists(ioManager.ConcatPath(model.Path, InstanceAttachFileName), cancellationToken).ConfigureAwait(false))
                {
                    return(Conflict(new ErrorMessage(ErrorCode.InstanceAtExistingPath)));
                }
                else
                {
                    attached = true;
                }
            }

            var newInstance = CreateDefaultInstance(model);

            DatabaseContext.Instances.Add(newInstance);
            try
            {
                await DatabaseContext.Save(cancellationToken).ConfigureAwait(false);

                try
                {
                    // actually reserve it now
                    await ioManager.CreateDirectory(unNormalizedPath, cancellationToken).ConfigureAwait(false);

                    await ioManager.DeleteFile(ioManager.ConcatPath(targetInstancePath, InstanceAttachFileName), cancellationToken).ConfigureAwait(false);
                }
                catch
                {
                    // oh shit delete the model
                    DatabaseContext.Instances.Remove(newInstance);

                    // DCT: Operation must always run
                    await DatabaseContext.Save(default).ConfigureAwait(false);
Example #29
0
        /// <summary>
        /// Run a quick DD instance to test the DMAPI is installed on the target code
        /// </summary>
        /// <param name="timeout">The timeout in seconds for validation</param>
        /// <param name="securityLevel">The <see cref="DreamDaemonSecurity"/> level to use to validate the API</param>
        /// <param name="job">The <see cref="Models.CompileJob"/> for the operation</param>
        /// <param name="byondLock">The current <see cref="IByondExecutableLock"/></param>
        /// <param name="portToUse">The port to use for API validation</param>
        /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
        /// <returns>A <see cref="Task"/> representing the running operation</returns>
        async Task VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, ushort portToUse, CancellationToken cancellationToken)
        {
            logger.LogTrace("Verifying DMAPI...");
            var launchParameters = new DreamDaemonLaunchParameters
            {
                AllowWebClient = false,
                PrimaryPort    = portToUse,
                SecurityLevel  = securityLevel,
                StartupTimeout = timeout
            };

            var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);

            job.MinimumSecurityLevel = securityLevel;             // needed for the TempDmbProvider
            var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout);

            using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Concat(job.DmeName, DmbExtension), job))
                using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false))
                {
                    var launchResult = await controller.LaunchResult.ConfigureAwait(false);

                    var now = DateTimeOffset.Now;
                    if (now < timeoutAt && launchResult.StartupTime.HasValue)
                    {
                        var timeoutTask = Task.Delay(timeoutAt - now, cancellationToken);

                        await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false);

                        cancellationToken.ThrowIfCancellationRequested();
                    }

                    if (controller.Lifetime.IsCompleted)
                    {
                        var validationStatus = controller.ApiValidationStatus;
                        logger.LogTrace("API validation status: {0}", validationStatus);
                        switch (validationStatus)
                        {
                        case ApiValidationStatus.RequiresUltrasafe:
                            job.MinimumSecurityLevel = DreamDaemonSecurity.Ultrasafe;
                            return;

                        case ApiValidationStatus.RequiresSafe:
                            if (securityLevel == DreamDaemonSecurity.Ultrasafe)
                            {
                                throw new JobException("This game must be run with at least the 'Safe' DreamDaemon security level!");
                            }
                            job.MinimumSecurityLevel = DreamDaemonSecurity.Safe;
                            return;

                        case ApiValidationStatus.RequiresTrusted:
                            if (securityLevel != DreamDaemonSecurity.Trusted)
                            {
                                throw new JobException("This game must be run with at least the 'Trusted' DreamDaemon security level!");
                            }
                            job.MinimumSecurityLevel = DreamDaemonSecurity.Trusted;
                            return;

                        case ApiValidationStatus.NeverValidated:
                            break;

                        case ApiValidationStatus.BadValidationRequest:
                            throw new JobException("Recieved an unrecognized API validation request from DreamDaemon!");

                        case ApiValidationStatus.UnaskedValidationRequest:
                        default:
                            throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Session controller returned unexpected ApiValidationStatus: {0}", validationStatus));
                        }
                    }

                    throw new JobException("DMAPI validation timed out!");
                }
        }
Example #30
0
        /// <summary>
        /// Installs a BYOND <paramref name="version"/> if it isn't already
        /// </summary>
        /// <param name="version">The BYOND <see cref="Version"/> to install</param>
        /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
        /// <returns>A <see cref="Task"/> representing the running operation</returns>
        async Task InstallVersion(Version version, CancellationToken cancellationToken)
        {
            var  ourTcs = new TaskCompletionSource <object>();
            Task inProgressTask;

            var  versionKey = VersionKey(version);
            bool installed;

            lock (installedVersions)
            {
                installed = installedVersions.TryGetValue(versionKey, out inProgressTask);
                if (!installed)
                {
                    installedVersions.Add(versionKey, ourTcs.Task);
                }
            }

            if (installed)
            {
                using (cancellationToken.Register(() => ourTcs.SetCanceled()))
                {
                    await Task.WhenAny(ourTcs.Task, inProgressTask).ConfigureAwait(false);

                    cancellationToken.ThrowIfCancellationRequested();
                    return;
                }
            }
            else
            {
                logger.LogDebug("Requested BYOND version {0} not currently installed. Doing so now...");
            }

            // okay up to us to install it then
            try
            {
                await eventConsumer.HandleEvent(EventType.ByondInstallStart, new List <string> {
                    versionKey
                }, cancellationToken).ConfigureAwait(false);

                var downloadTask = byondInstaller.DownloadVersion(version, cancellationToken);

                await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false);

                try
                {
                    var download = await downloadTask.ConfigureAwait(false);

                    await ioManager.CreateDirectory(versionKey, cancellationToken).ConfigureAwait(false);

                    var extractPath = ioManager.ResolvePath(versionKey);
                    logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath);
                    await ioManager.ZipToDirectory(extractPath, download, cancellationToken).ConfigureAwait(false);

                    await byondInstaller.InstallByond(extractPath, version, cancellationToken).ConfigureAwait(false);

                    // make sure to do this last because this is what tells us we have a valid version in the future
                    await ioManager.WriteAllBytes(ioManager.ConcatPath(versionKey, VersionFileName), Encoding.UTF8.GetBytes(version.ToString()), cancellationToken).ConfigureAwait(false);
                }
                catch (WebException e)
                {
                    // since the user can easily provide non-exitent version numbers, we'll turn this into a JobException
                    throw new JobException(ErrorCode.ByondDownloadFail, e);
                }
                catch (OperationCanceledException)
                {
                    throw;
                }
                catch
                {
                    await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false);

                    throw;
                }

                ourTcs.SetResult(null);
            }
            catch (Exception e)
            {
                if (!(e is OperationCanceledException))
                {
                    await eventConsumer.HandleEvent(EventType.ByondInstallFail, new List <string> {
                        e.Message
                    }, cancellationToken).ConfigureAwait(false);
                }
                lock (installedVersions)
                    installedVersions.Remove(versionKey);
                ourTcs.SetException(e);
                throw;
            }
        }