Esempio n. 1
0
        /// <inheritdoc />
        public async Task <ToolResult> GetDifferences(string mapPathA, string mapPathB, CancellationToken cancellationToken)
        {
            if (mapPathA == null)
            {
                throw new ArgumentNullException(nameof(mapPathA));
            }
            if (mapPathB == null)
            {
                throw new ArgumentNullException(nameof(mapPathB));
            }

            var        output      = new StringBuilder();
            var        errorOutput = new StringBuilder();
            var        args        = String.Format(CultureInfo.InvariantCulture, "{2}diff-maps \"{0}\" \"{1}\"", mapPathA, mapPathB, dmeArgument);
            Task <int> processTask;

            using (var P = CreateDMMToolsProcess(output, errorOutput))
            {
                P.StartInfo.Arguments = args;

                processTask = StartAndWaitForProcessExit(P, cancellationToken);
                await processTask.ConfigureAwait(false);
            }
            var toolOutput = String.Format(CultureInfo.InvariantCulture, "Exit Code: {0}{1}StdOut:{1}{2}{1}StdErr:{1}{3}", processTask.Result, Environment.NewLine, output, errorOutput);

            var result = new ToolResult
            {
                CommandLine = args,
                ToolOutput  = toolOutput
            };

            var matches = Regex.Matches(output.ToString(), "\\(([1-9][0-9]*), ([1-9][0-9]*), ([1-9][0-9]*)\\)");

            if (matches.Count == 0)
            {
                return(result);
            }

            var region = new MapRegion()
            {
                MinX = Int16.MaxValue,
                MinY = Int16.MaxValue
            };

            try
            {
                foreach (Match I in matches)
                {
                    region.MaxX = Math.Max(region.MaxX, Convert.ToInt16(I.Groups[1].Value, CultureInfo.InvariantCulture));
                    region.MinX = Math.Min(region.MinX, Convert.ToInt16(I.Groups[1].Value, CultureInfo.InvariantCulture));

                    region.MaxY = Math.Max(region.MaxY, Convert.ToInt16(I.Groups[2].Value, CultureInfo.InvariantCulture));
                    region.MinY = Math.Min(region.MinY, Convert.ToInt16(I.Groups[2].Value, CultureInfo.InvariantCulture));
                }
                result.MapRegion = region;
            }
            catch { }

            return(result);
        }
Esempio n. 2
0
        /// <inheritdoc />
        public async Task <ToolResult> GetMapSize(string mapPath, CancellationToken cancellationToken)
        {
            if (mapPath == null)
            {
                throw new ArgumentNullException(nameof(mapPath));
            }

            string     mapName;
            var        output      = new StringBuilder();
            var        errorOutput = new StringBuilder();
            var        args        = String.Format(CultureInfo.InvariantCulture, "{0}map-info -j \"{1}\"", dmeArgument, mapPath);
            Task <int> processTask;

            using (var P = CreateDMMToolsProcess(output, errorOutput))
            {
                P.StartInfo.Arguments = args;

                processTask = StartAndWaitForProcessExit(P, cancellationToken);

                mapName = Path.GetFileNameWithoutExtension(mapPath);

                await processTask.ConfigureAwait(false);
            }

            var toolOutput = String.Format(CultureInfo.InvariantCulture, "Exit Code: {0}{1}StdOut:{1}{2}{1}StdErr:{1}{3}", processTask.Result, Environment.NewLine, output, errorOutput);
            var result     = new ToolResult {
                ToolOutput = toolOutput, CommandLine = args
            };

            try
            {
                var json = JsonConvert.DeserializeObject <IDictionary <string, IDictionary <string, object> > >(output.ToString());
                var map  = json[mapPath];
                var size = (JArray)map["size"];
                result.MapRegion = new MapRegion
                {
                    MinX = 1,
                    MinY = 1,
                    MaxX = (short)size[0],
                    MaxY = (short)size[1]
                };
            }
            catch { }
            return(result);
        }
Esempio n. 3
0
        /// <summary>
        /// Generate map diffs for a given <paramref name="pullRequest"/>
        /// </summary>
        /// <param name="pullRequest">The <see cref="PullRequest"/></param>
        /// <param name="checkRunId">The <see cref="CheckRun.Id"/></param>
        /// <param name="changedDmms">Paths to changed .dmm files</param>
        /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
        /// <returns>A <see cref="Task"/> representing the running operation</returns>
        async Task GenerateDiffs(PullRequest pullRequest, long checkRunId, IReadOnlyList <string> changedDmms, CancellationToken cancellationToken)
        {
            using (logger.BeginScope("Generating {0} diffs for pull request #{1} in {2}/{3}", changedDmms.Count, pullRequest.Number, pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name))
            {
                const string OldMapExtension = ".old_map_diff_bot";

                var gitHubManager = serviceProvider.GetRequiredService <IGitHubManager>();

                Task generatingCommentTask;
                List <Task <RenderResult> > afterRenderings, beforeRenderings;

                var workingDir = ioManager.ConcatPath(pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name, pullRequest.Number.ToString(CultureInfo.InvariantCulture));
                logger.LogTrace("Setting workdir to {0}", workingDir);
                IIOManager currentIOManager = new ResolvingIOManager(ioManager, workingDir);
                string     repoPath;

                int  lastProgress       = -1;
                Task lastProgressUpdate = Task.CompletedTask;
                async Task OnCloneProgress(int progress)
                {
                    lock (gitHubManager)
                    {
                        if (lastProgress >= progress)
                        {
                            return;
                        }
                        if (lastProgress == -1)
                        {
                            logger.LogInformation("Waiting on repository to finish cloning...");
                        }
                        lastProgress = progress;
                    }
                    await lastProgressUpdate.ConfigureAwait(false);

                    await gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate
                    {
                        Status = CheckStatus.InProgress,
                        Output = new CheckRunOutput(stringLocalizer["Cloning Repository"], stringLocalizer["Clone Progress: {0}%", progress], null, null, null),
                    }, cancellationToken).ConfigureAwait(false);
                };
                Task CreateBlockedComment()
                {
                    logger.LogInformation("Waiting for another diff generation on {0}/{1} to complete...", pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name);
                    return(gitHubManager.CreateSingletonComment(pullRequest, stringLocalizer["Waiting for another operation on this repository to complete..."], cancellationToken));
                };

                logger.LogTrace("Locking repository...");
                using (var repo = await repositoryManager.GetRepository(pullRequest.Base.Repository, OnCloneProgress, CreateBlockedComment, cancellationToken).ConfigureAwait(false))
                {
                    logger.LogTrace("Repository ready");
                    generatingCommentTask = gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate
                    {
                        Status = CheckStatus.InProgress,
                        Output = new CheckRunOutput(stringLocalizer["Generating Diffs"], stringLocalizer["Aww geez rick, I should eventually put some progress message here"], null, null, null),
                    }, cancellationToken);
                    //prep the outputDirectory
                    async Task DirectoryPrep()
                    {
                        logger.LogTrace("Cleaning workdir...");
                        await currentIOManager.DeleteDirectory(".", cancellationToken).ConfigureAwait(false);

                        await currentIOManager.CreateDirectory(".", cancellationToken).ConfigureAwait(false);

                        logger.LogTrace("Workdir cleaned");
                    };

                    var dirPrepTask = DirectoryPrep();
                    //get the dme to use
                    var dmeToUseTask = serviceProvider.GetRequiredService <IDatabaseContext>().InstallationRepositories.Where(x => x.Id == pullRequest.Base.Repository.Id).Select(x => x.TargetDme).ToAsyncEnumerable().FirstOrDefault(cancellationToken);

                    var oldMapPaths = new List <string>()
                    {
                        Capacity = changedDmms.Count
                    };
                    try
                    {
                        //fetch base commit if necessary and check it out, fetch pull request
                        if (!await repo.ContainsCommit(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false))
                        {
                            logger.LogTrace("Base commit not found, running fetch...");
                            await repo.Fetch(cancellationToken).ConfigureAwait(false);
                        }
                        logger.LogTrace("Moving HEAD to pull request base...");
                        await repo.Checkout(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false);

                        //but since we don't need this right await don't await it yet
                        var pullRequestFetchTask = repo.FetchPullRequest(pullRequest.Number, cancellationToken);
                        try
                        {
                            //first copy all modified maps to the same location with the .old_map_diff_bot extension
                            async Task <string> CacheMap(string mapPath)
                            {
                                var originalPath = currentIOManager.ConcatPath(repoPath, mapPath);

                                if (await currentIOManager.FileExists(originalPath, cancellationToken).ConfigureAwait(false))
                                {
                                    logger.LogTrace("Creating old map cache of {0}", mapPath);
                                    var oldMapPath = String.Format(CultureInfo.InvariantCulture, "{0}{1}", originalPath, OldMapExtension);
                                    await currentIOManager.CopyFile(originalPath, oldMapPath, cancellationToken).ConfigureAwait(false);

                                    return(oldMapPath);
                                }
                                return(null);
                            };

                            repoPath = repo.Path;

                            var tasks = changedDmms.Select(x => CacheMap(x)).ToList();
                            await Task.WhenAll(tasks).ConfigureAwait(false);

                            oldMapPaths.AddRange(tasks.Select(x => x.Result));
                        }
                        finally
                        {
                            logger.LogTrace("Waiting for pull request commits to be available...");
                            await pullRequestFetchTask.ConfigureAwait(false);
                        }

                        logger.LogTrace("Creating and moving HEAD to pull request merge commit...");
                        //generate the merge commit ourselves since we can't get it from GitHub because itll return an outdated one
                        await repo.Merge(pullRequest.Head.Sha, cancellationToken).ConfigureAwait(false);
                    }
                    finally
                    {
                        logger.LogTrace("Waiting for configured project dme...");
                        await dmeToUseTask.ConfigureAwait(false);
                    }

                    //create empty array of map regions
                    var mapRegions = Enumerable.Repeat <MapRegion>(null, changedDmms.Count).ToList();
                    var dmeToUse   = dmeToUseTask.Result;

                    var generator       = generatorFactory.CreateGenerator(dmeToUse, new ResolvingIOManager(ioManager, repoPath));
                    var outputDirectory = currentIOManager.ResolvePath(".");
                    logger.LogTrace("Full workdir path: {0}", outputDirectory);
                    //Generate MapRegions for modified maps and render all new maps
                    async Task <RenderResult> DiffAndRenderNewMap(int I)
                    {
                        await dirPrepTask.ConfigureAwait(false);

                        var originalPath = currentIOManager.ConcatPath(repoPath, changedDmms[I]);

                        if (!await currentIOManager.FileExists(originalPath, cancellationToken).ConfigureAwait(false))
                        {
                            logger.LogTrace("No new map for path {0} exists, skipping region detection and after render", changedDmms[I]);
                            return(new RenderResult {
                                InputPath = changedDmms[I], ToolOutput = stringLocalizer["Map missing!"]
                            });
                        }
                        ToolResult result = null;

                        if (oldMapPaths[I] != null)
                        {
                            logger.LogTrace("Getting diff region for {0}...", changedDmms[I]);
                            result = await generator.GetDifferences(oldMapPaths[I], originalPath, cancellationToken).ConfigureAwait(false);

                            var region = result.MapRegion;
                            logger.LogTrace("Diff region for {0}: {1}", changedDmms[I], region);
                            if (region != null)
                            {
                                var       xdiam             = region.MaxX - region.MinX;
                                var       ydiam             = region.MaxY - region.MinY;
                                const int minDiffDimensions = 5 - 1;
                                if (xdiam < minDiffDimensions || ydiam < minDiffDimensions)
                                {
                                    //need to expand
                                    var fullResult = await generator.GetMapSize(originalPath, cancellationToken).ConfigureAwait(false);

                                    var fullRegion = fullResult.MapRegion;
                                    if (fullRegion == null)
                                    {
                                        //give up
                                        region = null;
                                    }
                                    else
                                    {
                                        bool increaseMax = true;
                                        if (xdiam < minDiffDimensions && ((fullRegion.MaxX - fullRegion.MinX) >= minDiffDimensions))
                                        {
                                            while ((region.MaxX - region.MinX) < minDiffDimensions)
                                            {
                                                if (increaseMax)
                                                {
                                                    region.MaxX = (short)Math.Min(region.MaxX + 1, fullRegion.MaxX);
                                                }
                                                else
                                                {
                                                    region.MinX = (short)Math.Max(region.MinX - 1, 1);
                                                }
                                                increaseMax = !increaseMax;
                                            }
                                        }
                                        if (ydiam < minDiffDimensions && ((fullRegion.MaxY - fullRegion.MinY) >= minDiffDimensions))
                                        {
                                            while ((region.MaxY - region.MinY) < minDiffDimensions)
                                            {
                                                if (increaseMax)
                                                {
                                                    region.MaxY = (short)Math.Min(region.MaxY + 1, fullRegion.MaxY);
                                                }
                                                else
                                                {
                                                    region.MinY = (short)Math.Max(region.MinY - 1, 1);
                                                }
                                                increaseMax = !increaseMax;
                                            }
                                        }
                                    }
                                    logger.LogTrace("Region for {0} expanded to {1}", changedDmms[I], region);
                                }
                                mapRegions[I] = region;
                            }
                        }
                        else
                        {
                            logger.LogTrace("Skipping region detection for {0} due to old map not existing", changedDmms[I]);
                        }
                        logger.LogTrace("Performing after rendering for {0}...", changedDmms[I]);
                        var renderResult = await generator.RenderMap(originalPath, mapRegions[I], outputDirectory, "after", cancellationToken).ConfigureAwait(false);

                        logger.LogTrace("After rendering for {0} complete! Result path: {1}, Output: {2}", changedDmms[I], renderResult.OutputPath, renderResult.ToolOutput);
                        if (result != null)
                        {
                            renderResult.ToolOutput = String.Format(CultureInfo.InvariantCulture, "Differences task:{0}{1}{0}Render task:{0}{2}", Environment.NewLine, result.ToolOutput, renderResult.ToolOutput);
                        }
                        return(renderResult);
                    };

                    logger.LogTrace("Running iterations of DiffAndRenderNewMap...");
                    //finish up before we go back to the base branch
                    afterRenderings = Enumerable.Range(0, changedDmms.Count).Select(I => DiffAndRenderNewMap(I)).ToList();
                    try
                    {
                        await Task.WhenAll(afterRenderings).ConfigureAwait(false);
                    }
                    catch (Exception e)
                    {
                        logger.LogDebug(e, "After renderings produced exception!");
                        //at this point everything is done but some have failed
                        //we'll handle it later
                    }

                    logger.LogTrace("Moving HEAD back to pull request base...");
                    await repo.Checkout(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false);

                    Task <RenderResult> RenderOldMap(int i)
                    {
                        var oldPath = oldMapPaths[i];

                        if (oldMapPaths != null)
                        {
                            logger.LogTrace("Performing before rendering for {0}...", changedDmms[i]);
                            return(generator.RenderMap(oldPath, mapRegions[i], outputDirectory, "before", cancellationToken));
                        }
                        return(Task.FromResult(new RenderResult {
                            InputPath = changedDmms[i], ToolOutput = stringLocalizer["Map missing!"]
                        }));
                    }

                    logger.LogTrace("Running iterations of RenderOldMap...");
                    //finish up rendering
                    beforeRenderings = Enumerable.Range(0, changedDmms.Count).Select(I => RenderOldMap(I)).ToList();
                    try
                    {
                        await Task.WhenAll(beforeRenderings).ConfigureAwait(false);
                    }
                    catch (Exception e)
                    {
                        logger.LogDebug(e, "Before renderings produced exception!");
                        //see above
                    }

                    //done with the repo at this point
                    logger.LogTrace("Renderings complete. Releasing reposiotory");
                }

                //collect results and errors
                async Task <KeyValuePair <MapDiff, MapRegion> > GetResult(int i)
                {
                    var beforeTask = beforeRenderings[i];
                    var afterTask  = afterRenderings[i];

                    var result = new MapDiff
                    {
                        InstallationRepositoryId = pullRequest.Base.Repository.Id,
                        CheckRunId = checkRunId,
                        FileId     = i,
                    };

                    RenderResult GetRenderingResult(Task <RenderResult> task)
                    {
                        if (task.Exception != null)
                        {
                            result.LogMessage = result.LogMessage == null?task.Exception.ToString() : String.Format(CultureInfo.InvariantCulture, "{0}{1}{2}", result.LogMessage, Environment.NewLine, task.Exception);

                            return(null);
                        }
                        return(task.Result);
                    };

                    var r1 = GetRenderingResult(beforeTask);
                    var r2 = GetRenderingResult(afterTask);

                    logger.LogTrace("Results for {0}: Before {1}, After {2}", changedDmms[i], r1?.OutputPath ?? "NONE", r2?.OutputPath ?? "NONE");

                    result.MapPath = changedDmms[i];

                    result.LogMessage = String.Format(CultureInfo.InvariantCulture, "Job {5}:{0}Path: {6}{0}Before:{0}Command Line: {1}{0}Output:{0}{2}{0}Logs:{0}{7}{0}After:{0}Command Line: {3}{0}Output:{0}{4}{0}Logs:{0}{8}{0}Exceptions:{0}{9}{0}", Environment.NewLine, r1?.CommandLine, r1?.OutputPath, r2?.CommandLine, r2?.OutputPath, i + 1, result.MapPath, r1?.ToolOutput, r2?.ToolOutput, result.LogMessage);

                    async Task <byte[]> ReadMapImage(string path)
                    {
                        if (path != null && await currentIOManager.FileExists(path, cancellationToken).ConfigureAwait(false))
                        {
                            var bytes = await currentIOManager.ReadAllBytes(path, cancellationToken).ConfigureAwait(false);

                            await currentIOManager.DeleteFile(path, cancellationToken).ConfigureAwait(false);

                            return(bytes);
                        }
                        return(null);
                    }

                    var readBeforeTask = ReadMapImage(r1?.OutputPath);

                    result.AfterImage = await ReadMapImage(r2?.OutputPath).ConfigureAwait(false);

                    result.BeforeImage = await readBeforeTask.ConfigureAwait(false);

                    return(new KeyValuePair <MapDiff, MapRegion>(result, r2?.MapRegion));
                }

                logger.LogTrace("Waiting for notification comment to POST...");
                await generatingCommentTask.ConfigureAwait(false);

                logger.LogTrace("Collecting results...");
                var results = Enumerable.Range(0, changedDmms.Count).Select(x => GetResult(x)).ToList();
                await Task.WhenAll(results).ConfigureAwait(false);

                var dic = new Dictionary <MapDiff, MapRegion>();
                foreach (var I in results.Select(x => x.Result))
                {
                    dic.Add(I.Key, I.Value);
                }
                await HandleResults(pullRequest, checkRunId, dic, cancellationToken).ConfigureAwait(false);
            }
        }