// The GraphQL Api does not support the recursive reading of files so using the V3 API
public async Task <List <RepositoryFile> > ReadFilesAsync(string owner, string name, string gitRef)
{
var repoFiles = new List <RepositoryFile>();
TreeResponse treeResponse = null;
try
{
treeResponse = await treesClient.GetRecursive(owner, name, gitRef).ConfigureAwait(false);
}
catch (Octokit.ApiException ex) when(ex.Message == "Git Repository is empty.")
{
return(repoFiles);
}
var treeItems = treeResponse.Tree;
if (treeItems != null && treeItems.Any())
{
var fileTreeItems = treeItems.Where(treeItem => treeItem.Type == TreeType.Blob);
foreach (var fileTreeItem in fileTreeItems)
{
var codeRepoFile = new RepositoryFile();
codeRepoFile.FullPath = fileTreeItem.Path;
codeRepoFile.Name = Path.GetFileName(codeRepoFile.FullPath);
repoFiles.Add(codeRepoFile);
}
}
return(repoFiles);
}
public async Task <List <GitFile> > GetFilesForCommitAsync(string repoUri, string commit, string path)
{
path = path.Replace('\\', '/');
path = path.TrimStart('/').TrimEnd('/');
(string owner, string repo) = ParseRepoUri(repoUri);
TreeResponse pathTree = await GetTreeForPathAsync(owner, repo, commit, path);
TreeResponse recursiveTree = await GetRecursiveTreeAsync(owner, repo, pathTree.Sha);
GitFile[] files = await Task.WhenAll(
recursiveTree.Tree.Where(treeItem => treeItem.Type == TreeType.Blob)
.Select(
async treeItem =>
{
Blob blob = await Client.Git.Blob.Get(owner, repo, treeItem.Sha);
return(new GitFile(
path + "/" + treeItem.Path,
blob.Content,
blob.Encoding == EncodingType.Base64 ? "base64" : "utf-8")
{
Mode = treeItem.Mode
});
}));
return(files.ToList());
}
/// <summary>
/// Retrieve a set of file under a specific path at a commit
/// </summary>
/// <param name="repoUri">Repository URI</param>
/// <param name="commit">Commit to get files at</param>
/// <param name="path">Path to retrieve files from</param>
/// <returns>Set of files under <paramref name="path"/> at <paramref name="commit"/></returns>
public async Task <List <GitFile> > GetFilesAtCommitAsync(string repoUri, string commit, string path)
{
path = path.Replace('\\', '/');
path = path.TrimStart('/').TrimEnd('/');
(string owner, string repo) = ParseRepoUri(repoUri);
if (string.IsNullOrEmpty(owner) || string.IsNullOrEmpty(repo))
{
_logger.LogInformation($"'owner' or 'repository' couldn't be inferred from '{repoUri}'. " +
$"Not getting files from 'eng/common...'");
return(new List <GitFile>());
}
TreeResponse pathTree = await GetTreeForPathAsync(owner, repo, commit, path);
TreeResponse recursiveTree = await GetRecursiveTreeAsync(owner, repo, pathTree.Sha);
GitFile[] files = await Task.WhenAll(
recursiveTree.Tree.Where(treeItem => treeItem.Type == TreeType.Blob)
.Select(
async treeItem =>
{
return(await GetGitTreeItem(path, treeItem, owner, repo));
}));
return(files.ToList());
}
public async Task PushFilesAsync(
List <GitFile> filesToCommit,
string repoUri,
string branch,
string commitMessage)
{
using (_logger.BeginScope("Pushing files to {branch}", branch))
{
(string owner, string repo) = ParseRepoUri(repoUri);
string baseCommitSha = await Client.Repository.Commit.GetSha1(owner, repo, branch);
Octokit.Commit baseCommit = await Client.Git.Commit.Get(owner, repo, baseCommitSha);
TreeResponse baseTree = await Client.Git.Tree.Get(owner, repo, baseCommit.Tree.Sha);
TreeResponse newTree = await CreateGitHubTreeAsync(owner, repo, filesToCommit, baseTree);
Octokit.Commit newCommit = await Client.Git.Commit.Create(
owner,
repo,
new NewCommit(commitMessage, newTree.Sha, baseCommit.Sha));
await Client.Git.Reference.Update(owner, repo, $"heads/{branch}", new ReferenceUpdate(newCommit.Sha));
}
}
JsonResult IDriver.Tree(string target)
{
var fullPath = ParsePath(target);
var answer = new TreeResponse();
if (fullPath.Directory.ResourceId.Equals(Guid.Empty))
{
// Volumen path
var volume = ResourceManager.Instance.GetOrAddVolumeFolder(_project, fullPath.Directory.TypesOfResource);
if (volume != null)
{
foreach (var item in volume.Children.OfType <ResourceFolder>())
{
answer.Tree.Add(DTOBase.Create(item, fullPath.Root));
}
}
}
else
{
foreach (var item in ResourceManager.Instance.GetFolders(fullPath.Directory))
{
answer.Tree.Add(DTOBase.Create(item, fullPath.Root));
}
}
return(Json(answer));
}
public async Task <List <GitFile> > GetFilesForCommitAsync(string repoUri, string commit, string path)
{
path = path.Replace('\\', '/');
path = path.TrimStart('/').TrimEnd('/');
(string owner, string repo) = ParseRepoUri(repoUri);
if (string.IsNullOrEmpty(owner) || string.IsNullOrEmpty(repo))
{
_logger.LogInformation($"'owner' or 'repository' couldn't be inferred from '{repoUri}'. " +
$"Not getting files from 'eng/common...'");
return(new List <GitFile>());
}
TreeResponse pathTree = await GetTreeForPathAsync(owner, repo, commit, path);
TreeResponse recursiveTree = await GetRecursiveTreeAsync(owner, repo, pathTree.Sha);
GitFile[] files = await Task.WhenAll(
recursiveTree.Tree.Where(treeItem => treeItem.Type == TreeType.Blob)
.Select(
async treeItem =>
{
Blob blob = await Client.Git.Blob.Get(owner, repo, treeItem.Sha);
return(new GitFile(
path + "/" + treeItem.Path,
blob.Content,
blob.Encoding == EncodingType.Base64 ? "base64" : "utf-8")
{
Mode = treeItem.Mode
});
}));
return(files.ToList());
}
public async Task <TreeResponse> getAllFiles()
{
//IReadOnlyList<RepositoryContent> files = await session.client.Repository.Content.GetAllContents(repo.Id, path);
TreeResponse files = await session.client.Git.Tree.GetRecursive(repo.Id, "master");
return(files);
}
JsonResult IDriver.Tree(string target)
{
FullPath fullPath = ParsePath(target);
TreeResponse answer = new TreeResponse();
foreach (var item in fullPath.Directory.GetDirectories())
{
answer.Tree.Add(DTOBase.Create(item, fullPath.Root));
}
return(Json(answer));
}
Tuple <Parts, TreeResponse> AddToPathCache(Parts parts, TreeResponse treeEntry)
{
if (treeCachePerPath.TryGetValue(parts.Url, out var treeFrom))
{
return(treeFrom);
}
treeFrom = new Tuple <Parts, TreeResponse>(parts, treeEntry);
treeCachePerPath.Add(parts.Url, treeFrom);
return(treeFrom);
}
private async Task <TreeResponse> GetRecursiveTreeAsync(string owner, string repo, string treeSha)
{
TreeResponse tree = await Client.Git.Tree.GetRecursive(owner, repo, treeSha);
if (tree.Truncated)
{
throw new NotSupportedException(
$"The git repository is too large for the github api. Getting recursive tree '{treeSha}' returned truncated results.");
}
return(tree);
}
/// <summary>
/// Retrieve a set of file under a specific path at a commit
/// </summary>
/// <param name="repoUri">Repository URI</param>
/// <param name="commit">Commit to get files at</param>
/// <param name="path">Path to retrieve files from</param>
/// <returns>Set of files under <paramref name="path"/> at <paramref name="commit"/></returns>
public async Task <List <GitFile> > GetFilesAtCommitAsync(string repoUri, string commit, string path)
{
path = path.Replace('\\', '/');
path = path.TrimStart('/').TrimEnd('/');
(string owner, string repo) = ParseRepoUri(repoUri);
if (string.IsNullOrEmpty(owner) || string.IsNullOrEmpty(repo))
{
_logger.LogInformation($"'owner' or 'repository' couldn't be inferred from '{repoUri}'. " +
$"Not getting files from 'eng/common...'");
return(new List <GitFile>());
}
TreeResponse pathTree = await GetTreeForPathAsync(owner, repo, commit, path);
TreeResponse recursiveTree = await GetRecursiveTreeAsync(owner, repo, pathTree.Sha);
GitFile[] files = await Task.WhenAll(
recursiveTree.Tree.Where(treeItem => treeItem.Type == TreeType.Blob)
.Select(
async treeItem =>
{
Blob blob = await ExponentialRetry.RetryAsync(
async() => await Client.Git.Blob.Get(owner, repo, treeItem.Sha),
ex => _logger.LogError(ex, $"Failed to get blob at sha {treeItem.Sha}"),
ex => ex is ApiException apiex && apiex.StatusCode >= HttpStatusCode.InternalServerError);
ContentEncoding encoding;
switch (blob.Encoding.Value)
{
case EncodingType.Base64:
encoding = ContentEncoding.Base64;
break;
case EncodingType.Utf8:
encoding = ContentEncoding.Utf8;
break;
default:
throw new NotImplementedException($"Unknown github encoding type {blob.Encoding.StringValue}");
}
return(new GitFile(
path + "/" + treeItem.Path,
blob.Content,
encoding)
{
Mode = treeItem.Mode
});
}));
return(files.ToList());
}
//async Task<List<string>> TraverseTreeManually(TreeResponse rootTree, GitHubClient github, Repository repository, string extension)
async Task <List <string> > TraverseTreeManually(TreeResponse rootTree, GitHubClient github, string owner, string repoName, string extension)
{
Stack <TreeItem> treeItems = new Stack <TreeItem>();
int count = 0;
List <string> topics = new List <string>();
var items = rootTree.Tree;
foreach (var item in items)
{
if (item.Type == TreeType.Blob)
{
var itemExtension = System.IO.Path.GetExtension(item.Path);
if (string.Equals(itemExtension, "." + extension, StringComparison.OrdinalIgnoreCase))
{
count++;
topics.Add(item.Path);
}
}
else if (item.Type == TreeType.Tree)
{
treeItems.Push(item);
}
}
while (treeItems.Count != 0)
{
var treeItem = treeItems.Pop();
var newTreeResponse = await github.GitDatabase.Tree.Get(owner, repoName, treeItem.Sha);
items = newTreeResponse.Tree;
foreach (var item in items)
{
if (item.Type == TreeType.Blob)
{
var itemExtension = System.IO.Path.GetExtension(item.Path);
if (string.Equals(itemExtension, "." + extension, StringComparison.OrdinalIgnoreCase))
{
count++;
topics.Add(item.Path);
}
}
else if (item.Type == TreeType.Tree)
{
treeItems.Push(item);
}
}
}
return(topics);
}
public JsonResult Tree(string target)
{
var path = Helper.DecodePath(target).Trim(Path.DirectorySeparatorChar);
var parentPath = string.IsNullOrEmpty(path) ? null : Path.GetDirectoryName(path);
var folders = mediaService.GetMediaFolders(path);
var answer = new TreeResponse();
foreach (var folder in folders)
{
answer.Tree.Add(CreateDto(folder, parentPath));
}
return(Json(answer));
}
public JsonResult Tree(string target)
{
FullPath fullPath = ParsePath(target);
TreeResponse answer = new TreeResponse();
foreach (var item in fullPath.Directory.GetDirectories())
{
if ((item.Attributes & FileAttributes.Hidden) != FileAttributes.Hidden)
{
answer.Tree.Add(DTOBase.Create(item, fullPath.Root));
}
}
return(Json(answer));
}
public static async Task <string> GetFileShaAsync(
this ITreesClient treesClient, string repoOwner, string repoName, string branch, string path)
{
string? dirPath = Path.GetDirectoryName(path)?.Replace("\\", "/");
TreeResponse treeResponse = await treesClient.Get(repoOwner, repoName, HttpUtility.UrlEncode($"{branch}:{dirPath}"));
string fileName = Path.GetFileName(path);
TreeItem?item = treeResponse.Tree
.FirstOrDefault(item => string.Equals(item.Path, fileName, StringComparison.OrdinalIgnoreCase));
if (item is null)
{
throw new InvalidOperationException(
$"Unable to find git tree data for path '{path}' in repo '{repoName}'.");
}
return(item.Sha);
}
JsonResult IDriver.Parents(string target)
{
WebDavRoot lroot = this.GetRoot(target);
target = this.GetCorectTarget(target);
target = this.DecodeTarget(target);
TreeResponse answer = new TreeResponse();
DirInfo dirInfo = client.GetInfo(target);
if (dirInfo == null)
{
throw new ElFinderFileNotExists();
}
DirInfo parent = this.GetParent(dirInfo);
if (parent == null)
{
answer.Tree.Add(DTOBase.Create(dirInfo, null, lroot));
}
else
{
List <DirInfo> directories = client.GetDirectories(parent.RelPath);
foreach (DirInfo d in directories)
{
if (!d.IsDirectory)
{
continue;
}
answer.Tree.Add(DTOBase.Create(d, parent, lroot));
}
while (parent.FullPath != lroot.Directory.FullPath)
{
var thisParent = (client.GetInfo(parent.getDir()));
//parent = (client.GetInfo(parent.getDir()));
answer.Tree.Add(DTOBase.Create(parent, thisParent, lroot));
parent = thisParent;
}
}
return(Json(answer));
}
static NewTree BuildNewTreeFrom(TreeResponse destinationParentTree)
{
var newTree = new NewTree();
foreach (var treeItem in destinationParentTree.Tree)
{
var newTreeItem = new NewTreeItem
{
Mode = treeItem.Mode,
Path = treeItem.Path,
Sha = treeItem.Sha,
Type = treeItem.Type.Value
};
newTree.Tree.Add(newTreeItem);
}
return(newTree);
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
TreeResponse res = (TreeResponse)value;
if (res.Request.RootName.HasValue())
{
writer.WriteStartArray();
writer.WriteStartObject();
writer.WritePropertyName("id");
writer.WriteValue(res.Request.RootId ?? "");
writer.WritePropertyName("text");
writer.WriteValue(res.Request.RootName);
if (res.Request.RootIcon.HasValue())
{
writer.WritePropertyName("iconCls");
writer.WriteValue(res.Request.RootIcon);
}
writer.WritePropertyName("children");
}
IEnumerable <DataRow> rows = null;
if (res.Request.Id.HasValue || !res.Data.Columns.Contains("parentid"))
{
rows = res.Data.Rows.Cast <DataRow>();
}
else if (res.Request.RootId.HasValue())
{
rows = res.Data.Select($"parentid='{res.Request.RootId}'", "orderno");
}
else
{
rows = res.Data.Select("parentid is null", "orderno");
}
WriteJson(writer, rows, res.Request);
if (res.Request.RootName.HasValue())
{
writer.WriteEndObject();
writer.WriteEndArray();
}
}
JsonResult IDriver.Tree(string target)
{
WebDavRoot lroot = this.GetRoot(target);
target = this.GetCorectTarget(target);
target = this.DecodeTarget(target);
TreeResponse answer = new TreeResponse();
List <DirInfo> directories = client.GetDirectories(target, true);
for (int i = 1; i < directories.Count; i++)
{
if (!directories[i].IsDirectory)
{
continue;
}
directories[i].HasSubDirectories = IsConstainsChild(directories[i]);
answer.Tree.Add(DTOBase.Create(directories[i], directories[0], lroot));
}
return(Json(answer));
}
private static async Task <NewTreeWithPath> CopySubtree(GitHubClient github, TreeResponse tree, long repoId, string excludePath, string treePath = "", int recursionLevel = 0)
{
NewTree newTree = new NewTree();
var trees = tree.Tree.Where(x => x.Type == TreeType.Tree);
var blobs = tree.Tree.Where(x => x.Type != TreeType.Tree);
blobs.Select(x => new NewTreeItem
{
Path = x.Path,
Mode = x.Mode,
Type = x.Type.Value,
Sha = x.Sha
}).ToList().ForEach(x => newTree.Tree.Add(x));
await(await trees.SelectAsync(async x => await CopySubtree(github, await github.Git.Tree.Get(repoId, x.Sha), repoId, excludePath, x.Path, recursionLevel + 1)))
.ToList().ForEachAsync(async x =>
{
NewTreeWithPath newTreeWithPath = x;
if (newTreeWithPath.Path == newTreeWithPath.ExcludePath)
{
return;
}
string newSha = (await github.Git.Tree.Create(repoId, newTreeWithPath.Tree)).Sha;
newTree.Tree.Add(new NewTreeItem
{
Mode = "040000",
Type = TreeType.Tree,
Sha = newSha,
Path = newTreeWithPath.Path
});
});
return(new NewTreeWithPath
{
Tree = newTree,
Path = treePath,
ExcludePath = recursionLevel > 1 ? string.Join('/', excludePath.Split("/").Skip(1)) : excludePath
});
}
private async Task <TreeResponse> GetTreeForPathAsync(string owner, string repo, string commitSha, string path)
{
var pathSegments = new Queue <string>(path.Split('/', '\\'));
var currentPath = new List <string>();
Octokit.Commit commit = await Client.Git.Commit.Get(owner, repo, commitSha);
string treeSha = commit.Tree.Sha;
while (true)
{
TreeResponse tree = await Client.Git.Tree.Get(owner, repo, treeSha);
if (tree.Truncated)
{
throw new NotSupportedException(
$"The git repository is too large for the github api. Getting tree '{treeSha}' returned truncated results.");
}
if (pathSegments.Count < 1)
{
return(tree);
}
string subfolder = pathSegments.Dequeue();
currentPath.Add(subfolder);
TreeItem subfolderItem = tree.Tree.Where(ti => ti.Type == TreeType.Tree)
.FirstOrDefault(ti => ti.Path == subfolder);
if (subfolderItem == null)
{
throw new DirectoryNotFoundException(
$"The path '{string.Join("/", currentPath)}' could not be found.");
}
treeSha = subfolderItem.Sha;
}
}
JsonResult IDriver.Parents(string target)
{
var fullPath = ParsePath(target);
var answer = new TreeResponse();
if (fullPath.Directory.FullName == fullPath.Root.Directory.FullName)
{
answer.Tree.Add(DTOBase.Create(fullPath.Directory, fullPath.Root));
}
else
{
var parent = fullPath.Directory;
foreach (var item in ResourceManager.Instance.GetFolders(parent))
{
answer.Tree.Add(DTOBase.Create(item, fullPath.Root));
}
while (parent != null && parent.FullName != fullPath.Root.Directory.FullName)
{
parent = parent.Parent as ResourceFolder;
answer.Tree.Add(DTOBase.Create(parent, fullPath.Root));
}
}
return(Json(answer));
}
JsonResult IDriver.Parents(string target)
{
FullPath fullPath = ParsePath(target);
TreeResponse answer = new TreeResponse();
if (fullPath.Directory.FullName == fullPath.Root.Directory.FullName)
{
answer.Tree.Add(DTOBase.Create(fullPath.Directory, fullPath.Root));
}
else
{
DirectoryInfo parent = fullPath.Directory;
foreach (var item in parent.Parent.GetDirectories())
{
answer.Tree.Add(DTOBase.Create(item, fullPath.Root));
}
while (parent.FullName != fullPath.Root.Directory.FullName)
{
parent = parent.Parent;
answer.Tree.Add(DTOBase.Create(parent, fullPath.Root));
}
}
return(Json(answer));
}
private async Task <ComponentTemplate> ResolveComponentTemplateAsync(ProjectTemplate projectTemplate, RepositoryReference repository, TreeResponse tree, TreeItem treeItem)
{
var componentYamlFiles = await client.Repository.Content
.GetAllContents(repository.Organization, repository.Repository, treeItem.Path)
.ConfigureAwait(false);
var componentYamlFile = componentYamlFiles.First();
var componentYaml = componentYamlFile.Content;
var componentJson = new Deserializer().ToJson(componentYaml);
var componentTemplate = TeamCloudSerialize.DeserializeObject <ComponentTemplate>(componentJson, new ComponentTemplateConverter(projectTemplate, treeItem.Url));
componentTemplate.Folder = Regex.Replace(treeItem.Path, $"/{Constants.ComponentYaml}$", string.Empty, RegexOptions.IgnoreCase);
componentTemplate.Description = await CheckAndPopulateFileContentAsync(repository, tree.Tree, componentTemplate.Description, componentTemplate.Folder).ConfigureAwait(false);
return(componentTemplate);
}
protected override async Task <IEnumerable <IDocument> > ExecuteConfigAsync(
IDocument input,
IExecutionContext context,
IMetadata values)
{
// See http://www.levibotelho.com/development/commit-a-file-with-the-github-api/
// Set up the client
GitHubClient github = new GitHubClient(new ProductHeaderValue("Statiq"), GitHubClient.GitHubApiUrl);
if (values.TryGetValue(Token, out string token))
{
github.Credentials = new Credentials(token);
}
else if (values.TryGetValue(Username, out string username) && values.TryGetValue(Password, out string password))
{
github.Credentials = new Credentials(username, password);
}
else
{
throw new ExecutionException("Could not determine GitHub credentials");
}
if (!values.TryGetValue(Owner, out string owner) ||
string.IsNullOrEmpty(owner) ||
!values.TryGetValue(Name, out string name) ||
string.IsNullOrEmpty(name))
{
throw new ExecutionException("Invalid repository owner or name");
}
// Get the current head tree
string branch = values.GetString(Branch, DefaultBranch);
Reference reference = await github.Git.Reference.Get(owner, name, "heads/" + branch);
Commit commit = await github.Git.Commit.Get(owner, name, reference.Object.Sha);
// Iterate the output path, adding new tree items
// Don't reference a base tree so that any items not reflected in the new tree will be deleted
NormalizedPath sourcePath = values.GetPath(SourcePath, context.FileSystem.GetOutputPath());
NewTree newTree = new NewTree();
foreach (string outputFile in Directory.GetFiles(sourcePath.FullPath, "*", SearchOption.AllDirectories))
{
// Upload the blob
BlobReference blob = await github.Git.Blob.Create(owner, name, new NewBlob
{
Content = Convert.ToBase64String(await File.ReadAllBytesAsync(outputFile)),
Encoding = EncodingType.Base64
});
// Add the new blob to the tree
string relativePath = Path.GetRelativePath(sourcePath.FullPath, outputFile).Replace("\\", "/");
newTree.Tree.Add(new NewTreeItem
{
Path = relativePath,
Mode = "100644",
Type = TreeType.Blob,
Sha = blob.Sha
});
}
// Create the new tree
TreeResponse newTreeResponse = await github.Git.Tree.Create(owner, name, newTree);
// Create the commit
Commit newCommit = await github.Git.Commit.Create(owner, name, new NewCommit($"Deployment from Statiq", newTreeResponse.Sha, commit.Sha));
// Update the head ref
await github.Git.Reference.Update(owner, name, reference.Ref, new ReferenceUpdate(newCommit.Sha, true));
return(await input.YieldAsync());
}
public async Task DeploySite()
{
StatusUpdate?.Invoke("Starting deploy...", StatusType.Status);
if (_githubToken == string.Empty)
{
// TODO: Send an error message
StatusUpdate?.Invoke("No GitHub token configured.", StatusType.Error);
return;
}
if (_repoName == string.Empty)
{
StatusUpdate?.Invoke("No repository configured.", StatusType.Error);
return;
}
string[] splitRepoName = _repoName.Split('/');
if (splitRepoName.Length != 2)
{
StatusUpdate?.Invoke("Repository name format invalid.", StatusType.Error);
return;
}
if (_repoDestination == string.Empty)
{
StatusUpdate?.Invoke("No destination configured.", StatusType.Error);
return;
}
StatusUpdate?.Invoke("Getting repository...", StatusType.Status);
long repoId = (await _github.Repository.Get(splitRepoName[0], splitRepoName[1])).Id;
string headMasterRef = "heads/" + _repoBranch;
Reference masterReference;
try
{
masterReference = await _github.Git.Reference.Get(repoId, headMasterRef);
}
catch (Exception ex)
{
if (ex is NotFoundException)
{
// This probably means the branch hasn't been created yet
// https://github.com/octokit/octokit.net/issues/1098
StatusUpdate?.Invoke("Creating branch " + _repoBranch + "...", StatusType.Status);
Reference master = await _github.Git.Reference.Get(repoId, "heads/master");
masterReference = await _github.Git.Reference.Create(repoId,
new NewReference("refs/" + headMasterRef, master.Object.Sha));
}
else
{
throw;
}
}
StatusUpdate?.Invoke("Getting current commit...", StatusType.Status);
GitHubCommit currentCommit = (await _github.Repository.Commit.GetAll(repoId)).First();
string currentTreeSha = currentCommit.Commit.Tree.Sha;
NewTree newTree = new NewTree();
if (_repoDirectory != string.Empty)
{
TreeResponse currentTree = await _github.Git.Tree.Get(repoId, currentTreeSha);
// Copy the current tree to the new tree
StatusUpdate?.Invoke("Copying current tree to new tree...", StatusType.Status);
newTree = (await CopySubtree(_github, currentTree, repoId, _repoDirectory)).Tree;
}
// Now, create all of the files in the tree
StatusUpdate?.Invoke("Uploading compiled pages and adding to tree...", StatusType.Status);
await _context.CompiledPages.ToList().ForEachAsync(async page =>
{
NewBlob newBlob = new NewBlob
{
Content = MiscUtils.Base64Encode(page.Contents),
Encoding = EncodingType.Base64
};
BlobReference newBlobCreated = await _github.Git.Blob.Create(repoId, newBlob);
string newBlobSha = newBlobCreated.Sha;
newTree.Tree.Add(new NewTreeItem
{
Mode = "100644",
Type = TreeType.Blob,
Sha = newBlobSha,
Path = Path.Join(_repoDirectory, page.Title + ".html")
});
});
// Upload all of the files from the wwwroot directory
StatusUpdate?.Invoke("Uploading other files and adding to tree...", StatusType.Status);
(await CopyDirectoryIntoTree(_github, repoId, _repoDirectory)).ForEach(item => { newTree.Tree.Add(item); });
StatusUpdate?.Invoke("Creating commit...", StatusType.Status);
string newTreeSha = (await _github.Git.Tree.Create(repoId, newTree)).Sha;
NewCommit newCommit = new NewCommit("Automated deploy from webweb", newTreeSha, masterReference.Object.Sha);
Commit createdCommit = await _github.Git.Commit.Create(repoId, newCommit);
await _github.Git.Reference.Update(repoId, headMasterRef, new ReferenceUpdate(createdCommit.Sha));
StatusUpdate?.Invoke("Deploy complete.", StatusType.Success);
}
/// <summary>
/// Uploads results to GitHub/Azure Table Storage
/// </summary>
/// <param name="scorecards">List of Scorecard instances to be uploaded</param>
/// <param name="githubClient">An authenticated Octokit.GitHubClient instance</param>
/// <param name="storageAccountKey">Key to the rollout scorecards storage account</param>
/// <param name="githubConfig">GitHubConfig object representing config</param>
public async static Task UploadResultsAsync(List <Scorecard> scorecards, GitHubClient githubClient, string storageAccountKey, GithubConfig githubConfig, bool skipPr = false)
{
// We batch the scorecards by date so they can be sorted into markdown files properly
IEnumerable <ScorecardBatch> scorecardBatches = scorecards
.GroupBy(s => s.Date).Select(g => new ScorecardBatch {
Date = g.Key, Scorecards = g.ToList()
});
const string TargetBranch = "main";
if (!skipPr)
{
Reference targetBranch = await githubClient.Git.Reference
.Get(githubConfig.ScorecardsGithubOrg, githubConfig.ScorecardsGithubRepo, "heads/" + TargetBranch);
string newBranchName = $"{DateTime.Today:yyyy-MM-dd}-Scorecard-Update";
string newBranchRef = $"heads/{newBranchName}";
Reference newBranch;
// If this succeeds than the branch exists and we should update it directly
try
{
newBranch = await githubClient.Git.Reference.Get(githubConfig.ScorecardsGithubOrg,
githubConfig.ScorecardsGithubRepo, newBranchRef);
}
// If not, we've got to create the new branch
catch (NotFoundException)
{
newBranch = await githubClient.Git.Reference.CreateBranch(githubConfig.ScorecardsGithubOrg,
githubConfig.ScorecardsGithubRepo, newBranchName, targetBranch);
}
TreeResponse currentTree = await githubClient.Git.Tree.Get(githubConfig.ScorecardsGithubOrg,
githubConfig.ScorecardsGithubRepo, newBranchRef);
NewTree newTree = new NewTree
{
BaseTree = currentTree.Sha,
};
// We loop over the batches and generate a markdown file for each rollout date
foreach (ScorecardBatch scorecardBatch in scorecardBatches)
{
List <RepoMarkdown> repoMarkdowns = scorecardBatch.Scorecards.Select(s => CreateRepoMarkdown(s)).ToList();
string scorecardBatchMarkdown = $"# {scorecardBatch.Date.Date:dd MMMM yyyy} Rollout Summaries\n\n" +
$"{string.Join('\n', repoMarkdowns.Select(md => md.Summary))}\n" +
$"# Itemized Scorecard\n\n" +
$"{string.Join('\n', repoMarkdowns.Select(md => md.Breakdown))}";
string scorecardBatchFilePath =
$"{githubConfig.ScorecardsDirectoryPath}Scorecard_{scorecardBatch.Date.Date:yyyy-MM-dd}.md";
NewTreeItem markdownBlob = new NewTreeItem
{
Path = scorecardBatchFilePath,
Mode = _gitFileBlobMode,
Type = TreeType.Blob,
Content = scorecardBatchMarkdown,
};
newTree.Tree.Add(markdownBlob);
}
TreeResponse treeResponse = await githubClient.Git.Tree.Create(githubConfig.ScorecardsGithubOrg, githubConfig.ScorecardsGithubRepo, newTree);
// Commit the new response to the new branch
NewCommit newCommit = new NewCommit("Add scorecards for " +
string.Join(", ", scorecardBatches.Select(s => s.Date.Date.ToString("yyyy-MM-dd"))),
treeResponse.Sha,
newBranch.Object.Sha);
Commit commit = await githubClient.Git.Commit
.Create(githubConfig.ScorecardsGithubOrg, githubConfig.ScorecardsGithubRepo, newCommit);
ReferenceUpdate update = new ReferenceUpdate(commit.Sha);
Reference updatedRef = await githubClient.Git.Reference.Update(githubConfig.ScorecardsGithubOrg,
githubConfig.ScorecardsGithubRepo, newBranchRef, update);
PullRequestRequest prRequest = new PullRequestRequest
{
Base = TargetBranch,
Head = newBranchName,
State = ItemStateFilter.Open,
};
// If an open PR exists already, we shouldn't try to create a new one
List <PullRequest> prs =
(await githubClient.PullRequest.GetAllForRepository(githubConfig.ScorecardsGithubOrg, githubConfig.ScorecardsGithubRepo)).ToList();
if (!prs.Any(pr => pr.Head.Ref == newBranchName))
{
NewPullRequest newPullRequest = new NewPullRequest(newCommit.Message, newBranchName, TargetBranch);
await githubClient.PullRequest.Create(githubConfig.ScorecardsGithubOrg, githubConfig.ScorecardsGithubRepo, newPullRequest);
}
}
// Upload the results to Azure Table Storage (will overwrite previous entries with new data if necessary)
CloudTable table = Utilities.GetScorecardsCloudTable(storageAccountKey);
foreach (Scorecard scorecard in scorecards)
{
ScorecardEntity scorecardEntity = new ScorecardEntity(scorecard.Date, scorecard.Repo.Repo)
{
TotalScore = scorecard.TotalScore,
TimeToRolloutSeconds = scorecard.TimeToRollout.TotalSeconds,
CriticalIssues = scorecard.CriticalIssues,
Hotfixes = scorecard.Hotfixes,
Rollbacks = scorecard.Rollbacks,
DowntimeSeconds = scorecard.Downtime.TotalSeconds,
Failure = scorecard.Failure,
TimeToRolloutScore = scorecard.TimeToRolloutScore,
CriticalIssuesScore = scorecard.CriticalIssueScore,
HotfixScore = scorecard.HotfixScore,
RollbackScore = scorecard.RollbackScore,
DowntimeScore = scorecard.DowntimeScore,
};
await table.ExecuteAsync(TableOperation.InsertOrReplace(scorecardEntity));
}
}
public async Task Apply(string owner, string repository, GhStandardFileSet fileSet)
{
Repository repo = await _client.Repository.Get(owner, repository);
// Determine if auto-content branch exists
string checkBranch = repo.DefaultBranch;
var branchName = _arguments.BranchName;
try
{
await _client.Repository.Branch.Get(repo.Id, branchName);
checkBranch = branchName;
Log.Information("{Repository}: branch '{BranchName}' exists, using that as base instead of '{DefaultBranch}'", repo.FullName, branchName, repo.DefaultBranch);
}
catch (NotFoundException)
{
}
// Diff files
List <string> upToDate = new List <string>();
Dictionary <string, byte[]> files = fileSet.GetFiles().ToDictionary(s => s.path, s => s.value);
{
foreach ((string path, byte[] value) in files)
{
try
{
byte[] existing = await _client.Repository.Content.GetRawContentByRef(owner, repository, path, checkBranch);
Utility.NormalizeNewlines(ref existing);
if (existing.SequenceEqual(value))
{
upToDate.Add(path);
}
}
catch (NotFoundException)
{
}
}
if (upToDate.Count == files.Count)
{
Log.Information("{Repository}: is up-to-date", repo.FullName);
return;
}
}
foreach (string path in files.Keys.Except(upToDate))
{
Log.Information("{Repository}: '{path}' is outdated", repo.FullName, path);
}
NewTree newTree = new NewTree();
foreach ((string path, byte[] value) in files)
{
BlobReference newBlob = await _client.Git.Blob.Create(repo.Id, new NewBlob
{
Content = Convert.ToBase64String(value),
Encoding = EncodingType.Base64
});
newTree.Tree.Add(new NewTreeItem
{
Type = TreeType.Blob,
Mode = "100644",
Path = path,
Sha = newBlob.Sha
});
}
Reference headReference = await _client.Git.Reference.Get(repo.Id, $"heads/{repo.DefaultBranch}");
string headCommit = headReference.Object.Sha;
TreeResponse previousTree = await _client.Git.Tree.Get(repo.Id, headReference.Ref);
newTree.BaseTree = previousTree.Sha;
TreeResponse newTreeResponse = await _client.Git.Tree.Create(repo.Id, newTree);
Commit createdCommit = await _client.Git.Commit.Create(repo.Id, new NewCommit("Updating standard content files for repository", newTreeResponse.Sha, headCommit)
{
Author = new Committer(_arguments.CommitAuthor, _arguments.CommitEmail, DateTimeOffset.UtcNow)
});
Reference existingBranch = null;
try
{
existingBranch = await _client.Git.Reference.Get(repo.Id, _branchNameRef);
}
catch (NotFoundException)
{
}
if (existingBranch != null)
{
Log.Information("{Repository}: Force-pushing to '{BranchName}'", repo.FullName, branchName);
// Update / force-push
await _client.Git.Reference.Update(repo.Id, _branchNameRef, new ReferenceUpdate(createdCommit.Sha, true));
}
else
{
Log.Information("{Repository}: Creating '{BranchName}'", repo.FullName, branchName);
// Create
await _client.Git.Reference.Create(repo.Id, new NewReference(_branchNameRef, createdCommit.Sha));
// Create PR
PullRequest newPr = await _client.Repository.PullRequest.Create(repo.Id, new NewPullRequest("Auto: Updating standardized files", branchName, repo.DefaultBranch));
Log.Information("{Repository}: PR created #{PrNumber} - {Title}", repo.FullName, newPr.Number, newPr.Title);
}
}
