public static async Task BuildFileTree(ILocalTempFileLocks tempFiles, ApplicationDbContext database,
LfsProject project, ILogger logger, CancellationToken cancellationToken)
{
var semaphore = tempFiles.GetTempFilePath($"gitFileTrees/{project.Slug}", out string tempPath);
await semaphore.WaitAsync(TimeSpan.FromMinutes(10), cancellationToken);
try
{
await GitRunHelpers.EnsureRepoIsCloned(project.CloneUrl, tempPath, true, cancellationToken);
try
{
await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
}
catch (Exception)
{
// In case the branch refers to a new branch
await GitRunHelpers.Fetch(tempPath, true, cancellationToken);
await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
}
await GitRunHelpers.Pull(tempPath, true, cancellationToken, true);
// Skip if commit has not changed
var newCommit = await GitRunHelpers.GetCurrentCommit(tempPath, cancellationToken);
if (newCommit == project.FileTreeCommit)
{
logger.LogDebug("Commit is still the same ({FileTreeCommit}), skipping tree update " +
"for {Id}",
project.FileTreeCommit, project.Id);
return;
}
logger.LogInformation("New commit {NewCommit} to build file tree from (previous: {FileTreeCommit}) " +
"for project {Id}", newCommit, project.FileTreeCommit, project.Id);
project.FileTreeCommit = newCommit;
// Make sure we don't have any extra files locally
await GitRunHelpers.Clean(tempPath, cancellationToken);
// And then make sure the DB file tree entries are fine
await UpdateFileTreeForProject(database, tempPath, project, cancellationToken);
}
finally
{
semaphore.Release();
}
project.FileTreeUpdated = DateTime.UtcNow;
await database.SaveChangesAsync(cancellationToken);
}
public async Task <IActionResult> ReceiveWebhook()
{
var hook = await database.GithubWebhooks.FindAsync(AppInfo.SingleResourceTableRowId);
if (hook == null)
{
logger.LogWarning("Github webhook secret is not configured, can't process webhook");
return(BadRequest("Incorrect secret"));
}
var payload = await CheckSignature(hook);
GithubWebhookContent data;
try
{
data = JsonSerializer.Deserialize <GithubWebhookContent>(payload,
new JsonSerializerOptions(JsonSerializerDefaults.Web)) ?? throw new NullDecodedJsonException();
if (data == null)
{
throw new Exception("deserialized value is null");
}
}
catch (Exception e)
{
logger.LogWarning("Error deserializing github webhook: {@E}", e);
throw new HttpResponseException()
{
Value = new BasicJSONErrorResult("Invalid content",
"Failed to deserialize payload").ToString()
};
}
if (!HttpContext.Request.Headers.TryGetValue("X-GitHub-Event", out StringValues typeHeader) ||
typeHeader.Count != 1)
{
throw new HttpResponseException()
{
Value = new BasicJSONErrorResult("Invalid request", "Missing X-GitHub-Event header").ToString()
};
}
var type = typeHeader[0];
// TODO: check type on these first two event detections as well
if (!string.IsNullOrEmpty(data.Ref) && data.RefType != "branch" && !string.IsNullOrEmpty(data.After))
{
// This is a push (commit)
logger.LogInformation("Received a push event for ref: {Ref}", data.Ref);
if (data.Deleted || data.After == AppInfo.NoCommitHash)
{
logger.LogInformation("Push was about a deleted thing");
}
else
{
if (data.Before == AppInfo.NoCommitHash)
{
logger.LogInformation(
"Received a push (probably a new branch) with no before set, setting to the after commit");
data.Before = data.After;
}
if (data.Repository == null)
{
throw new HttpResponseException()
{
Value = new BasicJSONErrorResult("Invalid request",
"Repository is needed for this event type").ToString(),
};
}
bool matched = false;
// Detect if this triggers any builds
foreach (var project in await database.CiProjects.Where(p =>
p.ProjectType == CIProjectType.Github && p.Enabled && !p.Deleted &&
p.RepositoryFullName == data.Repository.FullName).ToListAsync())
{
matched = true;
// Detect next id
var previousBuildId = await database.CiBuilds.Where(b => b.CiProjectId == project.Id)
.MaxAsync(b => (long?)b.CiBuildId) ?? 0;
var build = new CiBuild()
{
CiProjectId = project.Id,
CiBuildId = ++previousBuildId,
CommitHash = data.After,
RemoteRef = data.Ref,
Branch = GitRunHelpers.ParseRefBranch(data.Ref),
IsSafe = !GitRunHelpers.IsPullRequestRef(data.Ref),
PreviousCommit = data.Before,
CommitMessage = data.HeadCommit?.Message ?? data.Commits?.FirstOrDefault()?.Message,
ParsedCommits = data.Commits,
};
await database.CiBuilds.AddAsync(build);
await database.SaveChangesAsync();
jobClient.Enqueue <CheckAndStartCIBuild>(x =>
x.Execute(build.CiProjectId, build.CiBuildId, CancellationToken.None));
}
if (!matched)
{
logger.LogWarning("Push event didn't match any repos: {Fullname}", data.Repository.FullName);
}
}
}
else if (!string.IsNullOrEmpty(data.Ref))
{
// This is a branch push (or maybe a tag?)
}
else if (type == "pull_request")
{
bool matched = false;
if (data.Repository == null)
{
throw new HttpResponseException()
{
Value = new BasicJSONErrorResult("Invalid request",
"Repository is needed for this event type").ToString(),
};
}
if (data.PullRequest == null)
{
throw new HttpResponseException()
{
Value = new BasicJSONErrorResult("Invalid request",
"PullRequest data is needed for this event type").ToString(),
};
}
jobClient.Enqueue <CheckPullRequestStatusJob>(x => x.Execute(data.Repository.FullName,
data.PullRequest.Number, data.PullRequest.Head.Sha, data.PullRequest.User.Login,
!data.IsClosedPullRequest, CancellationToken.None));
// Detect if this PR is for any of our repos
foreach (var project in await database.CiProjects.Where(p =>
p.ProjectType == CIProjectType.Github && p.Enabled && !p.Deleted &&
p.RepositoryFullName == data.Repository.FullName).ToListAsync())
{
matched = true;
if (data.IsClosedPullRequest)
{
logger.LogInformation("A pull request was closed");
}
else if (data.Action == "synchronize" || data.Action == "opened")
{
// PR content was changed so we should rebuild (we don't react to other actions to avoid
// duplicate builds)
// TODO: CLA checks for PRs
// Queue a CLA check
// Only non-primary repo PRs have CI jobs ran on them as main repo commits trigger
// the push event
if (data.PullRequest.Head.Repo.Id != data.Repository.Id)
{
logger.LogInformation("Received pull request event from a fork: {FullName}",
data.PullRequest.Head.Repo.FullName);
var headRef = GitRunHelpers.GenerateRefForPullRequest(data.PullRequest.Number);
// Detect next id
var previousBuildId = await database.CiBuilds.Where(b => b.CiProjectId == project.Id)
.MaxAsync(b => (long?)b.CiBuildId) ?? 0;
var build = new CiBuild()
{
CiProjectId = project.Id,
CiBuildId = ++previousBuildId,
CommitHash = data.PullRequest.Head.Sha,
RemoteRef = headRef,
Branch = GitRunHelpers.ParseRefBranch(headRef),
IsSafe = false,
PreviousCommit = data.PullRequest.Base.Sha,
CommitMessage = $"Pull request #{data.PullRequest.Number}",
// TODO: commits would need to be retrieved from data.PullRequest.CommitsUrl
Commits = null,
};
await database.CiBuilds.AddAsync(build);
await database.SaveChangesAsync();
jobClient.Enqueue <CheckAndStartCIBuild>(x =>
x.Execute(build.CiProjectId, build.CiBuildId, CancellationToken.None));
}
// TODO: could run some special actions on PR open
if (data.Action == "opened")
{
}
}
}
if (!matched)
{
logger.LogWarning("Pull request event didn't match any repos: {Fullname}",
data.Repository.FullName);
}
}
// TODO: should this always be updated. Github might send us quite a few events if we subscribe to them all
hook.LastUsed = DateTime.UtcNow;
await database.SaveChangesAsync();
return(Ok());
}
public async Task Execute(long ciProjectId, long ciBuildId, CancellationToken cancellationToken)
{
var build = await database.CiBuilds.Include(c => c.CiProject)
.FirstOrDefaultAsync(c => c.CiProjectId == ciProjectId && c.CiBuildId == ciBuildId, cancellationToken);
if (build == null)
{
logger.LogError("Failed to find CIBuild to start");
return;
}
// Update our local repo copy and see what the wanted build config is
var semaphore = localTempFileLocks.GetTempFilePath($"ciRepos/{ciProjectId}", out string tempPath);
await semaphore.WaitAsync(cancellationToken);
var deserializer = new DeserializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
if (build.CiProject == null)
{
throw new NotLoadedModelNavigationException();
}
CiBuildConfiguration?configuration;
try
{
await GitRunHelpers.EnsureRepoIsCloned(build.CiProject.RepositoryCloneUrl, tempPath, true,
cancellationToken);
// Fetch the ref
await GitRunHelpers.FetchRef(tempPath, build.RemoteRef, cancellationToken);
// Then checkout the commit this build is actually for
await GitRunHelpers.Checkout(tempPath, build.CommitHash, true, cancellationToken, true);
// Clean out non-ignored files
await GitRunHelpers.Clean(tempPath, cancellationToken);
// Read the build configuration
var text = await File.ReadAllTextAsync(Path.Join(tempPath, AppInfo.CIConfigurationFile), Encoding.UTF8,
cancellationToken);
configuration = deserializer.Deserialize <CiBuildConfiguration>(text);
}
catch (Exception e)
{
configuration = null;
logger.LogError("Error when trying to read repository for starting jobs: {@E}", e);
}
finally
{
semaphore.Release();
}
if (configuration == null)
{
await CreateFailedJob(build, "Failed to read repository or build configuration", cancellationToken);
return;
}
// Check that configuration is valid
var errors = new List <ValidationResult>();
if (!Validator.TryValidateObject(configuration, new ValidationContext(configuration), errors))
{
logger.LogError("Build configuration object didn't pass validations, see following errors:");
foreach (var error in errors)
{
logger.LogError("Failure: {Error}", error);
}
// TODO: pass validation errors to the build output
await CreateFailedJob(build, "Invalid configuration yaml", cancellationToken);
return;
}
// TODO: refactor these checks to be cleaner
if (configuration.Jobs.Select(j => j.Value.Cache).Any(c =>
c.LoadFrom.Any(p => p.Contains("..") || p.StartsWith("/")) || c.WriteTo.Contains("..") ||
c.WriteTo.Contains('/') ||
(c.Shared != null && c.Shared.Any(s =>
s.Key.Contains("..") || s.Key.StartsWith("/") || s.Value.Contains("..") ||
s.Value.Contains('/'))) ||
(c.System != null && c.System.Any(s =>
s.Key.Contains("..") || s.Key.Contains('\'') || !s.Key.StartsWith("/") ||
s.Value.Contains("..") || s.Value.Contains('/') || s.Value.Contains('\'')))))
{
logger.LogError("Build configuration cache paths have \"..\" in them or starts with a slash");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden cache path", cancellationToken);
return;
}
if (configuration.Jobs.Select(j => j.Value.Image).Any(i => i.Contains("..") || i.StartsWith("/")))
{
logger.LogError("Build configuration image names have \"..\" in them or starts with a slash");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden image name", cancellationToken);
return;
}
if (configuration.Jobs.SelectMany(j => j.Value.Artifacts.Paths).Any(p =>
p.Length < 3 || p.Length > 250 || p.StartsWith("/") || p.Contains("..")))
{
logger.LogError("Build has a too long, short, or non-relative artifact path");
await CreateFailedJob(build, "Invalid configuration yaml, invalid artifact path(s)", cancellationToken);
return;
}
if (configuration.Jobs.Any(j => j.Key == "CLA"))
{
logger.LogError("Build configuration job contains 'CLA' in it");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden job name", cancellationToken);
return;
}
// TODO: do something with the version number here...
// Then queue the jobs we found in the configuration
var jobs = new List <CiJob>();
long jobId = 0;
foreach (var jobEntry in configuration.Jobs)
{
if (string.IsNullOrWhiteSpace(jobEntry.Key) || jobEntry.Key.Length > 80)
{
await CreateFailedJob(build, "Invalid job name in configuration", cancellationToken);
return;
}
var job = new CiJob()
{
CiProjectId = ciProjectId,
CiBuildId = ciBuildId,
CiJobId = ++jobId,
JobName = jobEntry.Key,
Image = jobEntry.Value.Image,
CacheSettingsJson = JsonSerializer.Serialize(jobEntry.Value.Cache),
};
await database.CiJobs.AddAsync(job, cancellationToken);
jobs.Add(job);
}
await database.SaveChangesAsync(cancellationToken);
// Send statuses to github
foreach (var job in jobs)
{
if (!await statusReporter.SetCommitStatus(build.CiProject.RepositoryFullName, build.CommitHash,
GithubAPI.CommitStatus.Pending, statusReporter.CreateStatusUrlForJob(job), "CI checks starting",
job.JobName))
{
logger.LogError("Failed to set commit status for a build's job: {JobName}", job.JobName);
}
}
// Queue remote executor check task which will allocate a server to run the job(s) on
jobClient.Enqueue <HandleControlledServerJobsJob>(x => x.Execute(CancellationToken.None));
}
