Example #1
0
 public static BuildResultInfo GetBuildResultInfo(this ModelBuild modelBuild) =>
 new BuildResultInfo(
     GetBuildAndDefinitionInfo(modelBuild),
     modelBuild.QueueTime,
     modelBuild.StartTime,
     modelBuild.FinishTime,
     modelBuild.BuildResult ?? BuildResult.None);
Example #2
0
        public async Task <ModelGitHubIssue> EnsureGitHubIssueAsync(ModelBuild modelBuild, GitHubIssueKey issueKey, bool saveChanges)
        {
            var query = GetModelBuildQuery(modelBuild.GetBuildKey())
                        .SelectMany(x => x.ModelGitHubIssues)
                        .Where(x =>
                               x.Number == issueKey.Number &&
                               x.Organization == issueKey.Organization &&
                               x.Repository == issueKey.Repository);
            var modelGitHubIssue = await query.SingleOrDefaultAsync().ConfigureAwait(false);

            if (modelGitHubIssue is object)
            {
                return(modelGitHubIssue);
            }

            modelGitHubIssue = new ModelGitHubIssue()
            {
                Organization = issueKey.Organization,
                Repository   = issueKey.Repository,
                Number       = issueKey.Number,
                ModelBuild   = modelBuild,
            };

            Context.ModelGitHubIssues.Add(modelGitHubIssue);

            if (saveChanges)
            {
                await Context.SaveChangesAsync().ConfigureAwait(false);
            }

            return(modelGitHubIssue);
        }
Example #3
0
        /// <summary>
        /// There are times when the AzDO API will not provide a Timeline for a Bulid. In those cases we still need to create
        /// a <see cref="ModelBuildAttempt"/> entry. The assumption is this is for the first attempt and all of the values will
        /// be
        /// </summary>
        /// <param name="modelBuild"></param>
        /// <param name="build"></param>
        /// <returns></returns>
        public async Task <ModelBuildAttempt> EnsureBuildAttemptWithoutTimelineAsync(ModelBuild modelBuild, Build build)
        {
            const int attempt           = 1;
            var       modelBuildAttempt = await Context.ModelBuildAttempts
                                          .Where(x => x.ModelBuildId == modelBuild.Id && x.Attempt == attempt)
                                          .FirstOrDefaultAsync().ConfigureAwait(false);

            if (modelBuildAttempt is object)
            {
                return(modelBuildAttempt);
            }

            modelBuildAttempt = new ModelBuildAttempt()
            {
                Attempt           = attempt,
                BuildResult       = build.Result,
                StartTime         = modelBuild.StartTime,
                FinishTime        = modelBuild.FinishTime,
                ModelBuild        = modelBuild,
                IsTimelineMissing = false,
            };
            Context.ModelBuildAttempts.Add(modelBuildAttempt);
            await Context.SaveChangesAsync().ConfigureAwait(false);

            return(modelBuildAttempt);
        }
Example #4
0
 public static BuildAndDefinitionInfo GetBuildAndDefinitionInfo(this ModelBuild modelBuild) =>
 new BuildAndDefinitionInfo(
     modelBuild.AzureOrganization,
     modelBuild.AzureProject,
     modelBuild.BuildNumber,
     modelBuild.DefinitionId,
     modelBuild.DefinitionName,
     GetGitHubBuildInfo(modelBuild));
Example #5
0
 public async Task EnsureResultAsync(ModelBuild modelBuild, Build build)
 {
     if (modelBuild.BuildResult != build.Result)
     {
         var buildInfo = build.GetBuildResultInfo();
         modelBuild.BuildResult = build.Result;
         modelBuild.StartTime   = buildInfo.StartTime;
         modelBuild.FinishTime  = buildInfo.FinishTime;
         await Context.SaveChangesAsync().ConfigureAwait(false);
     }
 }
Example #6
0
        public async Task <ModelBuild> EnsureBuildAsync(BuildResultInfo buildInfo)
        {
            var modelBuildId = GetModelBuildId(buildInfo.BuildKey);
            var modelBuild   = Context.ModelBuilds
                               .Where(x => x.Id == modelBuildId)
                               .FirstOrDefault();

            if (modelBuild is object)
            {
                // This code accounts for the fact that we will see multiple attempts of a build and that will
                // change the result. When those happens we should update all of the following values. It may
                // seem strange to update start and finish time here but that is how the AzDO APIs work and it's
                // best to model them in that way.
                if (modelBuild.BuildResult != buildInfo.BuildResult)
                {
                    modelBuild.StartTime   = buildInfo.StartTime;
                    modelBuild.FinishTime  = buildInfo.FinishTime;
                    modelBuild.BuildResult = buildInfo.BuildResult;
                    await Context.SaveChangesAsync().ConfigureAwait(false);
                }

                return(modelBuild);
            }

            var prKey = buildInfo.PullRequestKey;
            var modelBuildDefinition = await EnsureBuildDefinitionAsync(buildInfo.DefinitionInfo).ConfigureAwait(false);

            modelBuild = new ModelBuild()
            {
                Id = modelBuildId,
                ModelBuildDefinitionId = modelBuildDefinition.Id,
                AzureOrganization      = modelBuildDefinition.AzureOrganization,
                AzureProject           = modelBuildDefinition.AzureProject,
                GitHubOrganization     = buildInfo.GitHubBuildInfo?.Organization,
                GitHubRepository       = buildInfo.GitHubBuildInfo?.Repository,
                GitHubTargetBranch     = buildInfo.GitHubBuildInfo?.TargetBranch,
                PullRequestNumber      = prKey?.Number,
                StartTime      = buildInfo.StartTime,
                FinishTime     = buildInfo.FinishTime,
                QueueTime      = buildInfo.QueueTime,
                BuildNumber    = buildInfo.Number,
                BuildResult    = buildInfo.BuildResult,
                DefinitionName = buildInfo.DefinitionName,
                DefinitionId   = buildInfo.DefinitionInfo.Id,
            };
            Context.ModelBuilds.Add(modelBuild);
            Context.SaveChanges();
            return(modelBuild);
        }
Example #7
0
 public static DefinitionKey GetDefinitionKey(this ModelBuild modelBuild) =>
 new DefinitionKey(
     modelBuild.AzureOrganization,
     modelBuild.AzureProject,
     modelBuild.DefinitionId);
Example #8
0
 public static ModelBuildKind GetModelBuildKind(this ModelBuild modelBuild) =>
 TriageContextUtil.GetModelBuildKind(modelBuild.IsMergedPullRequest, modelBuild.PullRequestNumber);
Example #9
0
 public static GitHubBuildInfo GetGitHubBuildInfo(this ModelBuild modelBuild) =>
 new GitHubBuildInfo(
     modelBuild.GitHubOrganization,
     modelBuild.GitHubRepository,
     modelBuild.PullRequestNumber,
     modelBuild.GitHubTargetBranch);
Example #10
0
 public static BuildInfo GetBuildInfo(this ModelBuild modelBuild) =>
 new BuildInfo(
     modelBuild.AzureOrganization,
     modelBuild.AzureProject,
     modelBuild.BuildNumber,
     GetGitHubBuildInfo(modelBuild));
Example #11
0
 public static BuildKey GetBuildKey(this ModelBuild modelBuild) =>
 new BuildKey(
     modelBuild.AzureOrganization,
     modelBuild.AzureProject,
     modelBuild.BuildNumber);
Example #12
0
 public static GitHubPullRequestKey?GetGitHubPullRequestKey(ModelBuild build) =>
 build.PullRequestNumber.HasValue
         ? (GitHubPullRequestKey?)new GitHubPullRequestKey(build.GitHubOrganization, build.GitHubRepository, build.PullRequestNumber.Value)
         : null;
Example #13
0
        public async Task <ModelTestRun> EnsureTestRunAsync(ModelBuild modelBuild, int attempt, DotNetTestRun testRun, Dictionary <HelixInfo, HelixLogInfo> helixMap)
        {
            var modelTestRun = await FindModelTestRunAsync(modelBuild.GetBuildKey(), testRun.TestRun.Id).ConfigureAwait(false);

            if (modelTestRun is object)
            {
                return(modelTestRun);
            }

            var buildInfo = testRun.Build.GetBuildResultInfo();

            modelTestRun = new ModelTestRun()
            {
                AzureOrganization = buildInfo.Organization,
                AzureProject      = buildInfo.Project,
                ModelBuild        = modelBuild,
                TestRunId         = testRun.TestRun.Id,
                Name    = testRun.TestRun.Name,
                Attempt = attempt,
            };
            Context.ModelTestRuns.Add(modelTestRun);

            foreach (var dotnetTestCaseResult in testRun.TestCaseResults)
            {
                var testCaseResult = dotnetTestCaseResult.TestCaseResult;
                var testResult     = new ModelTestResult()
                {
                    TestFullName         = testCaseResult.TestCaseTitle,
                    Outcome              = testCaseResult.Outcome,
                    ModelTestRun         = modelTestRun,
                    ModelBuild           = modelBuild,
                    ErrorMessage         = testCaseResult.ErrorMessage,
                    IsSubResultContainer = testCaseResult.SubResults?.Length > 0,
                    IsSubResult          = false,
                };

                AddHelixInfo(testResult);
                Context.ModelTestResults.Add(testResult);

                if (testCaseResult.SubResults is { } subResults)
                {
                    foreach (var subResult in subResults)
                    {
                        var iterationTestResult = new ModelTestResult()
                        {
                            TestFullName         = testCaseResult.TestCaseTitle,
                            Outcome              = subResult.Outcome,
                            ModelTestRun         = modelTestRun,
                            ModelBuild           = modelBuild,
                            ErrorMessage         = subResult.ErrorMessage,
                            IsSubResultContainer = false,
                            IsSubResult          = true
                        };

                        AddHelixInfo(iterationTestResult);
                        Context.ModelTestResults.Add(iterationTestResult);
                    }
                }

                void AddHelixInfo(ModelTestResult testResult)
                {
                    if (dotnetTestCaseResult.HelixInfo is { } helixInfo&&
                        helixMap.TryGetValue(helixInfo, out var helixLogInfo))
                    {
                        testResult.IsHelixTestResult   = true;
                        testResult.HelixConsoleUri     = helixLogInfo.ConsoleUri;
                        testResult.HelixCoreDumpUri    = helixLogInfo.CoreDumpUri;
                        testResult.HelixRunClientUri   = helixLogInfo.RunClientUri;
                        testResult.HelixTestResultsUri = helixLogInfo.TestResultsUri;
                    }
                }
            }

            await Context.SaveChangesAsync().ConfigureAwait(false);

            return(modelTestRun);
        }
Example #14
0
        public async Task <ModelBuildAttempt> EnsureBuildAttemptAsync(ModelBuild modelBuild, BuildResult buildResult, Timeline timeline)
        {
            var attempt           = timeline.GetAttempt();
            var modelBuildAttempt = await Context.ModelBuildAttempts
                                    .Where(x => x.ModelBuildId == modelBuild.Id && x.Attempt == attempt)
                                    .FirstOrDefaultAsync().ConfigureAwait(false);

            if (modelBuildAttempt is object && !modelBuildAttempt.IsTimelineMissing)
            {
                return(modelBuildAttempt);
            }

            var startTimeQuery = timeline
                                 .Records
                                 .Where(x => x.Attempt == attempt)
                                 .Select(x => DevOpsUtil.ConvertFromRestTime(x.StartTime))
                                 .SelectNullableValue()
                                 .Select(x => (DateTime?)x.DateTime);
            var startTime = startTimeQuery.Any()
                ? startTimeQuery.Min()
                : modelBuild.StartTime;

            var finishTimeQuery = timeline
                                  .Records
                                  .Where(x => x.Attempt == attempt)
                                  .Select(x => DevOpsUtil.ConvertFromRestTime(x.FinishTime))
                                  .SelectNullableValue()
                                  .Select(x => (DateTime?)x.DateTime);
            var finishTime = finishTimeQuery.Any()
                ? finishTimeQuery.Max()
                : modelBuild.FinishTime;

            if (modelBuildAttempt is object)
            {
                modelBuildAttempt.BuildResult       = buildResult;
                modelBuildAttempt.StartTime         = startTime;
                modelBuildAttempt.FinishTime        = finishTime;
                modelBuildAttempt.IsTimelineMissing = false;
            }
            else
            {
                modelBuildAttempt = new ModelBuildAttempt()
                {
                    Attempt           = attempt,
                    BuildResult       = buildResult,
                    StartTime         = startTime,
                    FinishTime        = finishTime,
                    ModelBuild        = modelBuild,
                    IsTimelineMissing = false,
                };
                Context.ModelBuildAttempts.Add(modelBuildAttempt);
            }

            var timelineTree = TimelineTree.Create(timeline);

            foreach (var record in timeline.Records)
            {
                if (record.Issues is null ||
                    !timelineTree.TryGetJob(record, out var job))
                {
                    continue;
                }

                foreach (var issue in record.Issues)
                {
                    var timelineIssue = new ModelTimelineIssue()
                    {
                        Attempt           = attempt,
                        JobName           = job.Name,
                        RecordName        = record.Name,
                        TaskName          = record.Task?.Name ?? "",
                        RecordId          = record.Id,
                        Message           = issue.Message,
                        ModelBuild        = modelBuild,
                        IssueType         = issue.Type,
                        ModelBuildAttempt = modelBuildAttempt,
                    };
                    Context.ModelTimelineIssues.Add(timelineIssue);
                }
            }

            await Context.SaveChangesAsync().ConfigureAwait(false);

            return(modelBuildAttempt);
        }
Example #15
0
        private async Task RetryOsxDeprovisionAsync(ModelBuild modelBuild, List <ModelBuildAttempt> modelBuildAttempts)
        {
            Logger.LogInformation("Considering OSX deprovision retry");
            if (modelBuildAttempts.Count > 1)
            {
                Logger.LogInformation("Build already has multiple attempts");
                return;
            }

            var issues = await Context
                         .ModelTimelineIssues
                         .Where(x => x.ModelBuildId == modelBuild.Id)
                         .Select(x => new
            {
                x.Message,
                x.IssueType,
                x.JobName
            })
                         .ToListAsync()
                         .ConfigureAwait(false);

            var count = 0;

            foreach (var issue in issues)
            {
                if (issue.Message.Contains("Received request to deprovision: The request was cancelled by the remote provider", StringComparison.OrdinalIgnoreCase))
                {
                    count++;
                }
            }

            if (count == 0)
            {
                Logger.LogInformation("No OSX failures");
                return;
            }

            var jobFailedCount = issues
                                 .Where(x => x.IssueType == IssueType.Error)
                                 .GroupBy(x => x.JobName)
                                 .Count();

            if (jobFailedCount - count >= 4)
            {
                Logger.LogInformation("Too many non-OSX failures");
                return;
            }

            Logger.LogInformation("Retrying");
            await Server.RetryBuildAsync(modelBuild.AzureProject, modelBuild.BuildNumber).ConfigureAwait(false);

            var model = new ModelOsxDeprovisionRetry()
            {
                OsxJobFailedCount = count,
                JobFailedCount    = jobFailedCount,
                ModelBuild        = modelBuild,
            };

            Context.ModelOsxDeprovisionRetry.Add(model);
            await Context.SaveChangesAsync().ConfigureAwait(false);
        }