public static async Task BuildFileTree(ILocalTempFileLocks tempFiles, ApplicationDbContext database,
                                               LfsProject project, ILogger logger, CancellationToken cancellationToken)
        {
            var semaphore = tempFiles.GetTempFilePath($"gitFileTrees/{project.Slug}", out string tempPath);

            await semaphore.WaitAsync(TimeSpan.FromMinutes(10), cancellationToken);

            try
            {
                await GitRunHelpers.EnsureRepoIsCloned(project.CloneUrl, tempPath, true, cancellationToken);

                try
                {
                    await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
                }
                catch (Exception)
                {
                    // In case the branch refers to a new branch
                    await GitRunHelpers.Fetch(tempPath, true, cancellationToken);

                    await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
                }

                await GitRunHelpers.Pull(tempPath, true, cancellationToken, true);

                // Skip if commit has not changed
                var newCommit = await GitRunHelpers.GetCurrentCommit(tempPath, cancellationToken);

                if (newCommit == project.FileTreeCommit)
                {
                    logger.LogDebug("Commit is still the same ({FileTreeCommit}), skipping tree update " +
                                    "for {Id}",
                                    project.FileTreeCommit, project.Id);
                    return;
                }

                logger.LogInformation("New commit {NewCommit} to build file tree from (previous: {FileTreeCommit}) " +
                                      "for project {Id}", newCommit, project.FileTreeCommit, project.Id);

                project.FileTreeCommit = newCommit;

                // Make sure we don't have any extra files locally
                await GitRunHelpers.Clean(tempPath, cancellationToken);

                // And then make sure the DB file tree entries are fine
                await UpdateFileTreeForProject(database, tempPath, project, cancellationToken);
            }
            finally
            {
                semaphore.Release();
            }

            project.FileTreeUpdated = DateTime.UtcNow;
            await database.SaveChangesAsync(cancellationToken);
        }
Esempio n. 2
0
        public async Task Execute(long ciProjectId, long ciBuildId, CancellationToken cancellationToken)
        {
            var build = await database.CiBuilds.Include(c => c.CiProject)
                        .FirstOrDefaultAsync(c => c.CiProjectId == ciProjectId && c.CiBuildId == ciBuildId, cancellationToken);

            if (build == null)
            {
                logger.LogError("Failed to find CIBuild to start");
                return;
            }

            // Update our local repo copy and see what the wanted build config is
            var semaphore = localTempFileLocks.GetTempFilePath($"ciRepos/{ciProjectId}", out string tempPath);

            await semaphore.WaitAsync(cancellationToken);

            var deserializer = new DeserializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance)
                               .Build();

            if (build.CiProject == null)
            {
                throw new NotLoadedModelNavigationException();
            }

            CiBuildConfiguration?configuration;

            try
            {
                await GitRunHelpers.EnsureRepoIsCloned(build.CiProject.RepositoryCloneUrl, tempPath, true,
                                                       cancellationToken);

                // Fetch the ref
                await GitRunHelpers.FetchRef(tempPath, build.RemoteRef, cancellationToken);

                // Then checkout the commit this build is actually for
                await GitRunHelpers.Checkout(tempPath, build.CommitHash, true, cancellationToken, true);

                // Clean out non-ignored files
                await GitRunHelpers.Clean(tempPath, cancellationToken);

                // Read the build configuration
                var text = await File.ReadAllTextAsync(Path.Join(tempPath, AppInfo.CIConfigurationFile), Encoding.UTF8,
                                                       cancellationToken);

                configuration = deserializer.Deserialize <CiBuildConfiguration>(text);
            }
            catch (Exception e)
            {
                configuration = null;
                logger.LogError("Error when trying to read repository for starting jobs: {@E}", e);
            }
            finally
            {
                semaphore.Release();
            }

            if (configuration == null)
            {
                await CreateFailedJob(build, "Failed to read repository or build configuration", cancellationToken);

                return;
            }

            // Check that configuration is valid
            var errors = new List <ValidationResult>();

            if (!Validator.TryValidateObject(configuration, new ValidationContext(configuration), errors))
            {
                logger.LogError("Build configuration object didn't pass validations, see following errors:");

                foreach (var error in errors)
                {
                    logger.LogError("Failure: {Error}", error);
                }

                // TODO: pass validation errors to the build output
                await CreateFailedJob(build, "Invalid configuration yaml", cancellationToken);

                return;
            }

            // TODO: refactor these checks to be cleaner
            if (configuration.Jobs.Select(j => j.Value.Cache).Any(c =>
                                                                  c.LoadFrom.Any(p => p.Contains("..") || p.StartsWith("/")) || c.WriteTo.Contains("..") ||
                                                                  c.WriteTo.Contains('/') ||
                                                                  (c.Shared != null && c.Shared.Any(s =>
                                                                                                    s.Key.Contains("..") || s.Key.StartsWith("/") || s.Value.Contains("..") ||
                                                                                                    s.Value.Contains('/'))) ||
                                                                  (c.System != null && c.System.Any(s =>
                                                                                                    s.Key.Contains("..") || s.Key.Contains('\'') || !s.Key.StartsWith("/") ||
                                                                                                    s.Value.Contains("..") || s.Value.Contains('/') || s.Value.Contains('\'')))))
            {
                logger.LogError("Build configuration cache paths have \"..\" in them or starts with a slash");

                await CreateFailedJob(build, "Invalid configuration yaml, forbidden cache path", cancellationToken);

                return;
            }

            if (configuration.Jobs.Select(j => j.Value.Image).Any(i => i.Contains("..") || i.StartsWith("/")))
            {
                logger.LogError("Build configuration image names have \"..\" in them or starts with a slash");

                await CreateFailedJob(build, "Invalid configuration yaml, forbidden image name", cancellationToken);

                return;
            }

            if (configuration.Jobs.SelectMany(j => j.Value.Artifacts.Paths).Any(p =>
                                                                                p.Length < 3 || p.Length > 250 || p.StartsWith("/") || p.Contains("..")))
            {
                logger.LogError("Build has a too long, short, or non-relative artifact path");

                await CreateFailedJob(build, "Invalid configuration yaml, invalid artifact path(s)", cancellationToken);

                return;
            }

            if (configuration.Jobs.Any(j => j.Key == "CLA"))
            {
                logger.LogError("Build configuration job contains 'CLA' in it");

                await CreateFailedJob(build, "Invalid configuration yaml, forbidden job name", cancellationToken);

                return;
            }

            // TODO: do something with the version number here...

            // Then queue the jobs we found in the configuration
            var  jobs  = new List <CiJob>();
            long jobId = 0;

            foreach (var jobEntry in configuration.Jobs)
            {
                if (string.IsNullOrWhiteSpace(jobEntry.Key) || jobEntry.Key.Length > 80)
                {
                    await CreateFailedJob(build, "Invalid job name in configuration", cancellationToken);

                    return;
                }

                var job = new CiJob()
                {
                    CiProjectId       = ciProjectId,
                    CiBuildId         = ciBuildId,
                    CiJobId           = ++jobId,
                    JobName           = jobEntry.Key,
                    Image             = jobEntry.Value.Image,
                    CacheSettingsJson = JsonSerializer.Serialize(jobEntry.Value.Cache),
                };

                await database.CiJobs.AddAsync(job, cancellationToken);

                jobs.Add(job);
            }

            await database.SaveChangesAsync(cancellationToken);

            // Send statuses to github
            foreach (var job in jobs)
            {
                if (!await statusReporter.SetCommitStatus(build.CiProject.RepositoryFullName, build.CommitHash,
                                                          GithubAPI.CommitStatus.Pending, statusReporter.CreateStatusUrlForJob(job), "CI checks starting",
                                                          job.JobName))
                {
                    logger.LogError("Failed to set commit status for a build's job: {JobName}", job.JobName);
                }
            }

            // Queue remote executor check task which will allocate a server to run the job(s) on
            jobClient.Enqueue <HandleControlledServerJobsJob>(x => x.Execute(CancellationToken.None));
        }