public RepositoryContentWithCommitInfo([NotNull] RepositoryContent content, [CanBeNull] GitHubCommit commit = null, [CanBeNull] string message = null, [CanBeNull] DateTime?editTime = null)
{
Content = content;
Commit = commit;
_CommitMessage = message;
LastEditTime = editTime;
}
private async Task FillInfoByLogin(GitHubCommit commit, Data data)
{
var author = commit.Author;
if (author == null)
{
var fullName = commit.Commit.Author.Name.Split(" ");
var firstName = GetEmptyOrName(0, fullName);
var lastName = GetEmptyOrName(1, fullName);
data.CommitAuthorFirstName = firstName;
data.CommitAuthorLastName = lastName;
data.CommitAuthorLogin = "";
data.DevID = GetIntIdByString(data.CommitAuthorEmail + firstName + lastName);
}
else
{
var login = author.Login;
var user = await GetUser(login);
var fullName = user.Name.Split(" ");
var firstName = GetEmptyOrName(0, fullName);
var lastName = GetEmptyOrName(1, fullName);
data.CommitAuthorFirstName = firstName;
data.CommitAuthorLastName = lastName;
data.CommitAuthorLogin = login;
//data.DevID = author.Id;
data.DevID = GetIntIdByString(data.CommitAuthorEmail + firstName + lastName);
}
}
/// <summary>
/// Handles creating an Event object from a GitHub commit which will be streamed into Splunk.
/// </summary>
/// <param name="githubCommit">The individual GithubCommit object which holds the data for a commit.</param>
/// <param name="eventWriter">The EventWriter for streaming events to Splunk.</param>
/// <param name="owner">The GitHub repository owner's name.</param>
/// <param name="repositoryName">The GitHub repository's name.</param>
public async Task StreamCommit(GitHubCommit githubCommit, EventWriter eventWriter, string owner, string repositoryName)
{
string authorName = githubCommit.Commit.Author.Name;
DateTime date = githubCommit.Commit.Author.Date;
// Replace any newlines with a space
string commitMessage = Regex.Replace(githubCommit.Commit.Message, "\\n|\\r", " ");
dynamic json = new JObject();
json.sha = githubCommit.Sha;
json.api_url = githubCommit.Url;
json.url = "http://github.com/" + owner + "/" + repositoryName + "/commit/" + githubCommit.Sha;
json.message = commitMessage;
json.author = authorName;
json.date = date.ToString();
var commitEvent = new Event();
commitEvent.Stanza = repositoryName;
commitEvent.SourceType = "github_commits";
commitEvent.Time = date;
commitEvent.Data = json.ToString(Formatting.None);
await eventWriter.QueueEventForWriting(commitEvent);
}
public static async Task <string> GetUpdateText(string path)
{
GitHubClient client = GetClient(path);
UpdaterSettings options = UpdaterSettings.Load(path);
IReadOnlyList <Release> releases = await client.Repository.Release.GetAll(Settings.Default.RepositoryOwner, Settings.Default.RepositoryName);
IReadOnlyList <GitHubCommit> commits = await client.Repository.Commit.GetAll(Settings.Default.RepositoryOwner, Settings.Default.RepositoryName);
IEnumerable <Release> latestReleases = releases.OrderByDescending(c => c.CreatedAt).Take(15);
StringBuilder commitMessage = new StringBuilder();
foreach (Release release in latestReleases.Where(e => options.InstallPrereleases || !e.Prerelease))
{
string releaseMessage = release.Body;
if (string.IsNullOrEmpty(releaseMessage))
{
GitHubCommit commit = commits.FirstOrDefault(c => c.Commit.Sha == release.TargetCommitish);
releaseMessage = commit?.Commit.Message ?? "Unknown";
}
commitMessage.AppendLine($"{release.CreatedAt.Date.Date.ToShortDateString()}:");
commitMessage.AppendLine(releaseMessage);
commitMessage.AppendLine();
}
return(commitMessage.ToString());
}
public async Task <int> UpdateGitHubActionPullRequestCommits(string clientId, string clientSecret, TableStorageAuth tableStorageAuth,
string owner, string repo, string pull_number)
{
GitHubAPIAccess api = new GitHubAPIAccess();
JArray items = await api.GetGitHubPullRequestCommitsJArray(clientId, clientSecret, owner, repo, pull_number);
int itemsAdded = 0;
TableStorageCommonDA tableDA = new TableStorageCommonDA(tableStorageAuth, tableStorageAuth.TableGitHubPRCommits);
//Check each build to see if it's in storage, adding the items not in storage
foreach (JToken item in items)
{
GitHubCommit commit = JsonConvert.DeserializeObject <GitHubCommit>(item.ToString());
string partitionKey = CreateGitHubPRCommitPartitionKey(owner, repo, pull_number);
string rowKey = commit.sha;
AzureStorageTableModel newItem = new AzureStorageTableModel(partitionKey, rowKey, item.ToString());
if (await tableDA.AddItem(newItem) == true)
{
itemsAdded++;
}
}
return(itemsAdded);
}
public static async Task <string> GetCommitAuthorAsync(
string repositoryUrl,
string commit,
string personalAccessToken)
{
(string owner, string repoName) = ParseRepoUri(repositoryUrl);
Octokit.GitHubClient client = new Octokit.GitHubClient(new ProductHeaderValue("assets-publisher"));
Credentials tokenAuth = new Credentials(personalAccessToken);
client.Credentials = tokenAuth;
GitHubCommit commitInfo = await client.Repository.Commit.Get(owner, repoName, commit);
while (commitInfo.Author.Type == "Bot")
{
if (!commitInfo.Parents.Any())
{
break;
}
commit = commitInfo.Parents.First().Sha;
commitInfo = await client.Repository.Commit.Get(owner, repoName, commit);
}
return($"@{commitInfo.Author.Login}");
}
private GitHubCommentModel CreateCommentModel(int version, GitHubComment comment)
{
// create URL for the commit from the comment URL
string commentUrl = comment.html_url.AbsoluteUri;
int hashIndex = commentUrl.IndexOf('#');
string commitUrl = hashIndex == -1 ? commentUrl : commentUrl.Substring(0, hashIndex);
// create basic model
GitHubCommentModel model = new GitHubCommentModel
{
CommentUrl = commentUrl,
CommitUrl = commitUrl,
CommentBody = new HtmlString(comment.body_html),
CommitId = comment.commit_id.Substring(0, 8),
FilePath = comment.path,
LineNumber = comment.line.GetValueOrDefault() == 0 ? null : comment.line,
Commenter = comment.user.login,
};
// add extra details if present
if (version == 2)
{
GitHubCommit commit = m_commits[comment.commit_id];
string author = commit.author != null ? commit.author.login : "******";
model.Author = author;
model.CommitMessage = commit.commit.message;
model.CommitFiles = commit.files.Select(f => f.filename).OrderBy(f => f, StringComparer.OrdinalIgnoreCase).ToList();
}
return(model);
}
private bool TryMemeCommitMessage(GitHubCommit commit, ChannelState chnl)
{
List<string> topLineList = commit.Commit.Message.Split(' ').ToList<string>();
List<string> bottomLineList = new List<string>();
string topLineString = commit.Commit.Message;
string bottomLineString = "";
Channel channel = _client.GetChannel(chnl.ChannelID);
while ((topLineString.Length - bottomLineString.Length) > 7 && (topLineString.Count() > 1))
{
bottomLineList.Add(topLineList.Last());
topLineList.Remove(topLineList.Last());
bottomLineString = string.Join(" ",bottomLineList.ToArray());
topLineString = string.Join(" ", topLineList.ToArray());
if ((topLineString.Length - bottomLineString.Length) <= 7)
{
bottomLineList.Reverse();
bottomLineString = string.Join(" ",bottomLineList.ToArray());
Image imageWithMemeText = MemeGeneratingModule.PlaceImageText(topLineString, bottomLineString, "github");
string fileName = "Memes/Memmit" + DateTime.Now.ToString("hhmmss") + ".png";
imageWithMemeText.Save(fileName, ImageFormat.Png);
channel.SendFile(fileName);
System.Threading.Thread.Sleep(5000);
return true;
}
}
return false;
}
public static string ToByStr(this GitHubCommit commit)
{
if (commit.Author != null)
{
return(commit.Author.ToStr("by"));
}
return(commit.Commit.Author != null?commit.Commit.Author.ToStr("by") : "");
}
public CommitViewModel Init(string repositoryOwner, string repositoryName, string node, GitHubCommit commit = null)
{
RepositoryOwner = repositoryOwner;
RepositoryName = repositoryName;
Commit = commit;
Node = node;
return(this);
}
private string GetCommitInfo(GitHubCommit commit)
{
var commitAlias = commit.Commits.Count > 1
? "commits"
: "commit";
return($"**{commit.Commits.Count} new {commitAlias}** pushed to " +
$"[{commit.Branch}]({commit.CommitUrl})");
}
public CommitViewModel Init(string repositoryOwner, string repositoryName, string node, GitHubCommit commit = null, bool showRepository = false)
{
RepositoryOwner = repositoryOwner;
RepositoryName = repositoryName;
Commit = commit;
Node = node;
ShowRepository = showRepository;
return(this);
}
private async Task <HttpStatusCode> PushToSpace(GitHubCommit commit)
{
if (commit == null)
{
return(HttpStatusCode.InternalServerError);
}
return(await _spaceService.PushGitHubNotification(commit));
}
public async Task <HttpStatusCode> PushGitHubNotification(GitHubCommit commit)
{
var url = _urlBuilder.BuildChannelTextMessageUrl("2PTicn3FhRAU");
var content = GetContent(commit);
using (var client = _httpClientFactory.CreateClient("spaceHttpClient")) {
var result = await client.PostAsync(url, content);
return(result.StatusCode);
}
}
public static GitHubCommit GetNewestCommit()
{
using (WebClient client = new WebClient())
{
client.Headers.Add("User-Agent", "XIVMon");
var result =
client.DownloadString($"https://api.github.com/repos/{Repo}/commits");
return(GitHubCommit.FromJson(result)[0]);
}
}
public GhCommit(GitHubCommit commit)
{
Message = commit.Commit.Message;
if (commit.Author != null)
{
Author = commit.Author.Id;
}
if (commit.Files != null)
{
Files = commit.Files.Select(x => new CommitFile(x)).ToArray();
}
}
/**
* Check if file is binary or not
* return bool
*/
public async Task <bool> isFileBinary(string[] fileData, GitHubCommit commit)
{
IReadOnlyList <RepositoryContent> contents;
try
{
// Check if file is binary
contents = await client.Repository.Content
.GetAllContentsByRef(UserName, RepoName, fileData[0], commit.Sha).ConfigureAwait(false);
var targetFile = contents[0];
// Download file
string con;
using (var wc = new WebClient())
con = wc.DownloadString(targetFile.DownloadUrl);
// Check if file is binary
for (int i = 1; i < 512 && i < con.Length; i++)
{
if (con[i] == 0x00 && con[i - 1] == 0x00)
{
return(true);
}
}
}
catch
{ // Chck if file is binary for deleted or renamed files
contents = await client.Repository.Content
.GetAllContentsByRef(UserName, RepoName, fileData[0], fileData[1]).ConfigureAwait(false);
var targetFile = contents[0];
string con;
using (var wc = new WebClient())
con = wc.DownloadString(targetFile.DownloadUrl);
// Check if file is binary
for (int i = 1; i < 512 && i < con.Length; i++)
{
if (con[i] == 0x00 && con[i - 1] == 0x00)
{
return(true);
}
}
}
return(false);
}
public void UpdateFiles(GitHubCommit commit, DirectoryInfo temp_folder)
{
if (temp_folder.GetDirectories().Length == 0)
{
UpdateFiles(commit.Sha, temp_folder);
}
else
{
UpdateFiles(commit.Sha, temp_folder);
DirectoryInfo[] folders = temp_folder.GetDirectories();
for (int i = 0; i < folders.Length; i++)
{
UpdateFiles(commit, folders[i]);
}
}
}
private Release ExtractCommits(Repository repository,
string currentTag, string previousTag, Release rel, string testPattern, List <Contributor> users)
{
var @base = previousTag;
var head = currentTag;
var commits = new List <GitHubCommit>();
var response = repository.Client.Repository.Commit.Compare(repository.Owner, repository.Name,
@base, head).Result;
rel.NumberofPullRequests = response.Commits.Count;
var allIds = new List <int>();
Parallel.ForEach(response.Commits, (c) =>
{
GitHubCommit cmt = null;
try
{
cmt = repository.Client.Repository.Commit.Get(repository.Owner,
repository.Name, c.Sha).Result;
Console.WriteLine("Commit Id: " + cmt.NodeId);
}
catch (Exception ex)
{
Console.WriteLine("Unable to fetch commit : " + ex.Message);
}
if (cmt != null)
{
rel.NumberOfAdditions += cmt.Stats.Additions;
rel.NumberOfDeletions += cmt.Stats.Deletions;
rel.NumberOfFilesModified += cmt.Files.Count;
rel.TestFilesChanged += cmt.Files.Count(f => f.BlobUrl.ToLowerInvariant().Contains(testPattern));
rel.CommentsCount += cmt.Commit.CommentCount;
if (cmt.Author != null)
{
allIds.Add(cmt.Author.Id);
rel.PriorContributions += users.FirstOrDefault(u => u.Author.Id == cmt.Author.Id) != null ?
users.FirstOrDefault(u => u.Author.Id == cmt.Author.Id).Total : 0;
// rel.TotalFollowers += client.User. et(cmt.Author.Login).Result.Followers;
}
}
});
rel.NumberOfUniqueContributers = allIds.Distinct().Count();
rel.TestCoverageScore = Math.Abs(rel.NumberOfFilesModified - rel.TestFilesChanged);
return(rel);
}
private static void ValidateLocalRepository(GitHubCommit expectedCommit)
{
var root = DirectoryLayout.DetermineRootDirectory();
using (var repo = new LibGit2Sharp.Repository(root))
{
string tip = repo.Head.Tip.Sha;
if (tip != expectedCommit.Sha)
{
throw new UserErrorException($"Current local commit: {tip}. Aborting.");
}
var status = repo.RetrieveStatus("apis/apis.json");
if (status != FileStatus.Unaltered)
{
throw new UserErrorException($"Expected apis.json to be unaltered. Current status: {status}. Aborting.");
}
}
}
public string BuildChannelCommitMessage(GitHubCommit commit, string channelId)
{
var template = _messageTemplateBuilder.BuildCommitMessageTemplate(@"JsonTemplates\spaceMessage.json");
template.Recipient.Channel.Id = channelId;
template.Content.Sections[0].Elements[0].Content = commit.Pusher.Name;
template.Content.Sections[0].Elements[1].Content = GetCommitInfo(commit);
template.Content.Sections[0].Elements[2].Content = GetCommits(commit.Commits);
return(JsonConvert.SerializeObject(
template,
Formatting.Indented,
new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
}
));
}
private async Task <Data> GetData(GitHubCommit commit, string owner, string name)
{
var data = new Data
{
CommitID = GetIntIdByString(commit.Sha),
CommitSha = commit.Sha,
CommitAuthorEmail = commit.Commit.Author.Email,
CreatedAt = commit.Commit.Author.Date,
RepositoryName = owner + "/" + name,
};
await FillInfoByLogin(commit, data);
var currentCommit = await client.Repository.Commit.Get(owner, name, data.CommitSha);
var nameAndSha = currentCommit.Files
.Select(x => new NameAndSha
{
Name = x.Filename,
Id = GetIntIdByString(x.Sha)
});
data.NameAndSha = nameAndSha;
int additions = 0;
int deletions = 0;
var additionsArrayByCommit = currentCommit.Files.Select(x => x.Additions);
var deletionsArrayByCommit = currentCommit.Files.Select(x => x.Deletions);
foreach (var a in additionsArrayByCommit)
{
additions += a;
}
foreach (var d in deletionsArrayByCommit)
{
deletions += d;
}
data.Additions = additions;
data.Deletions = deletions;
return(data);
}
public async void AnnounceCommitMessage(GitHubCommit commit)
{
foreach (ChannelState chnl in Beta.ChannelStateRepository.ChannelStates)
{
if (commit.Commit.Message.Length < 110 && chnl.ChatBattleEnabled)
{
if (!TryMemeCommitMessage(commit, chnl)) return;
}
else
{
string msg = "Looks like a new commit was added!\n";
msg += "``` " + commit.Commit.Message + "```";
if (chnl.ChatBattleEnabled)
{
MessageQueue.Add(new QueuedMessage(msg, chnl.ChannelID));
}
}
}
}
public static GitHubCommitResponse ParseResponse(SocialHttpResponse response)
{
// Parse the raw JSON response
JsonObject obj = response.GetBodyAsJsonObject();
// Check for any errors
if (response.StatusCode != HttpStatusCode.OK)
{
string message = obj.GetString("message");
string url = obj.GetString("documentation_url");
throw new GitHubHttpException(response.StatusCode, message, url);
}
// Initialize the object to be returned
return(new GitHubCommitResponse(response)
{
Data = GitHubCommit.Parse(obj)
});
}
public static async Task <Embed> GetLatestBuild()
{
GitHubClient github = new GitHubClient(new ProductHeaderValue("Vita3KBot"));
Release latestRelease = await github.Repository.Release.Get("Vita3k", "Vita3k", "continous");
ReleaseAsset linuxRelease = latestRelease.Assets.Where(release => {
return(release.Name.StartsWith("ubuntu-latest"));
}).First();
ReleaseAsset macosRelease = latestRelease.Assets.Where(release => {
return(release.Name.StartsWith("macos-latest"));
}).First();
ReleaseAsset windowsRelease = latestRelease.Assets.Where(release => {
return(release.Name.StartsWith("windows-latest"));
}).First();
string commit = latestRelease.Body.Substring(latestRelease.Body.IndexOf(":") + 1).Trim();
commit = commit.Substring(0, commit.Length - 1);
GitHubCommit REF = await github.Repository.Commit.Get("Vita3k", "Vita3k", commit);
Issue prInfo = await GetPRInfo(github, commit);
EmbedBuilder LatestBuild = new EmbedBuilder();
if (prInfo != null)
{
LatestBuild.WithTitle($"PR: #{prInfo.Number} By {prInfo.User.Login}")
.WithUrl(prInfo.HtmlUrl);
}
else
{
LatestBuild.WithTitle($"Commit: {REF.Sha} By {REF.Commit.Author.Name}")
.WithUrl($"https://github.com/vita3k/vita3k/commit/{REF.Sha}");
}
LatestBuild.WithDescription($"{REF.Commit.Message}")
.WithColor(Color.Orange)
.AddField("Windows", $"[{windowsRelease.Name}]({windowsRelease.BrowserDownloadUrl})")
.AddField("Linux", $"[{linuxRelease.Name}]({linuxRelease.BrowserDownloadUrl})")
.AddField("Mac", $"[{macosRelease.Name}]({macosRelease.BrowserDownloadUrl})");
return(LatestBuild.Build());
}
internal CommitItemViewModel(GitHubCommit commit, Action <CommitItemViewModel> action)
{
var msg = commit?.Commit?.Message ?? string.Empty;
var firstLine = msg.IndexOf("\n", StringComparison.Ordinal);
var description = firstLine > 0 ? msg.Substring(0, firstLine) : msg;
_description = new Lazy <string>(() => Emojis.FindAndReplace(description));
var time = DateTimeOffset.MinValue;
if (commit.Commit.Committer != null)
{
time = commit.Commit.Committer.Date;
}
Time = time.UtcDateTime.Humanize();
Name = commit.GenerateCommiterName();
Avatar = new GitHubAvatar(commit.GenerateGravatarUrl());
Commit = commit;
GoToCommand = ReactiveCommand.Create().WithSubscription(_ => action(this));
}
private async Task CreateReleasesAsync(List <ApiMetadata> newReleases, GitHubCommit commit)
{
var originalMessage = commit.Commit.Message;
var unwrappedMessage = string.Join("\n", UnwrapLines(originalMessage.Split('\n')));
Console.WriteLine("Creating releases with tags:");
foreach (var api in newReleases)
{
var tag = $"{api.Id}-{api.Version}";
var gitRelease = new NewRelease(tag)
{
Prerelease = !api.IsReleaseVersion,
Name = $"{api.Id} version {api.Version}",
TargetCommitish = commit.Sha,
Body = unwrappedMessage
};
// We could parallelize, but there's very little point.
await _client.Repository.Release.Create(RepositoryOwner, RepositoryName, gitRelease);
Console.WriteLine(tag);
}
}
private async Task <HttpStatusCode> UpdateLastCommitDateAsync(GitHubCommit lastCommit)
{
using var command = dbConnection.CreateCommand();
command.Parameters.AddWithValue("@id", 0);
command.Parameters.AddWithValue("@data", lastCommit.Commit.Author.Date.ToStringWithSeparatorAndZone(CultureInfo.InvariantCulture));
command.Parameters.AddWithValue("@sha", lastCommit.Sha);
command.CommandText = "update [lastcommit] set [data] = @data, [sha] = @sha where[id] = @id";
dbConnection.Open();
int rows = await command.ExecuteNonQueryAsync();
if (rows == 1)
{
dbConnection.Close();
return(HttpStatusCode.OK);
}
dbConnection.Close();
return(HttpStatusCode.BadRequest);
}
public CommitControl(GitHubCommit commit)
{
Commit = commit;
Wrapper = new Panel();
Wrapper.AutoSize = true;
Wrapper.Margin = new Padding(0, 0, 0, 15);
// create new label instance
HeaderLabel = new System.Windows.Forms.Label();
HeaderLabel.AutoSize = true;
if (commit.Commit.Message.IndexOf('\n') > -1)
{
HeaderLabel.Text = commit.Commit.Message.Substring(0, commit.Commit.Message.IndexOf('\n'));
}
else
{
HeaderLabel.Text = commit.Commit.Message;
}
HeaderLabel.Font = new Font(new FontFamily(System.Drawing.Text.GenericFontFamilies.Serif), 12, FontStyle.Bold);
Wrapper.Controls.Add(HeaderLabel);
// insertions and deletitions labels
InsertionsCountLabel = new System.Windows.Forms.Label();
InsertionsCountLabel.AutoSize = true;
InsertionsCountLabel.ForeColor = Color.DarkGreen;
InsertionsCountLabel.Text = "+" + commit.Stats.Additions.ToString();
InsertionsCountLabel.Location = new Point(0, HeaderLabel.Height);
Wrapper.Controls.Add(InsertionsCountLabel);
DeletionsCountLabel = new System.Windows.Forms.Label();
DeletionsCountLabel.AutoSize = true;
DeletionsCountLabel.ForeColor = Color.DarkRed;
DeletionsCountLabel.Text = "-" + commit.Stats.Deletions.ToString();
DeletionsCountLabel.Location = new Point(0, HeaderLabel.Height + InsertionsCountLabel.Height);
Wrapper.Controls.Add(DeletionsCountLabel);
}
public static async Task <IEnumerable <RepositoryContentWithCommitInfo> > TryLoadLinkedCommitDataAsync(
[NotNull] Task <IReadOnlyList <RepositoryContent> > contentTask, [NotNull] String htmlUrl,
[NotNull] GitHubClient client, long repoId, [NotNull] String branch, CancellationToken token)
{
// Try to download the file info
IReadOnlyList <RepositoryContent> contents = null;
try
{
// Load the full HTML body
WebView view = new WebView();
TaskCompletionSource <String> tcs = new TaskCompletionSource <String>();
view.NavigationCompleted += (s, e) =>
{
view.InvokeScriptAsync("eval", new[] { "document.documentElement.outerHTML;" }).AsTask().ContinueWith(t =>
{
tcs.SetResult(t.Status == TaskStatus.RanToCompletion ? t.Result : null);
});
};
// Manually set the user agent to get the full desktop site
String userAgent = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; ARM; Trident/7.0; Touch; rv:11.0; WPDesktop) like Gecko";
HttpRequestMessage httpRequestMessage = new HttpRequestMessage(HttpMethod.Post, new Uri(htmlUrl));
httpRequestMessage.Headers.Append("User-Agent", userAgent);
view.NavigateWithHttpRequestMessage(httpRequestMessage);
// Run the web calls in parallel
await Task.WhenAll(contentTask, tcs.Task);
contents = contentTask.Result;
String html = tcs.Task.Result;
if (token.IsCancellationRequested)
{
return(contents?.OrderByDescending(entry => entry.Type).Select(content => new RepositoryContentWithCommitInfo(content)));
}
// Load the HTML document
HtmlDocument document = new HtmlDocument();
document.LoadHtml(html);
/* =================
* HTML error tags
* =================
* ...
* <include-fragment class="commit-tease commit-loader">
* ...
* <div class="loader-error"/>
* </include-fragment>
* ... */
// Check if the HTML loading was successful
if (document.DocumentNode
?.Descendants("include-fragment") // Get the <include-fragment/> nodes
?.FirstOrDefault(node => node.Attributes?.AttributesWithName("class") // Get the nodes with a class attribute
?.FirstOrDefault(att => att.Value?.Equals("commit-tease commit-loader") == true) // That attribute must have this name
!= null) // There must be a node with these specs if the HTML loading failed
?.Descendants("div") // Get the inner <div/> nodes
?.FirstOrDefault(node => node.Attributes?.AttributesWithName("class")?.FirstOrDefault() // Check the class name
?.Value?.Equals("loader-error") == true) != null || // Make sure there was in fact a loading error
html.Contains("class=\"warning include - fragment - error\"") ||
html.Contains("Failed to load latest commit information"))
{
System.Diagnostics.Debug.WriteLine("[DEBUG] Fallback");
// Use the Oktokit APIs to get the info
IEnumerable <Task <IReadOnlyList <GitHubCommit> > > tasks = contents.Select(r => client.Repository.Commit.GetAll(repoId,
new CommitRequest {
Path = r.Path, Sha = branch
}, // Only get the commits that edited the current file
new ApiOptions {
PageCount = 1, PageSize = 1
})); // Just get the latest commit for this file
IReadOnlyList <GitHubCommit>[] commits = await Task.WhenAll(tasks);
// Query the results
return(contents.AsParallel().OrderByDescending(file => file.Type).Select((file, i) =>
{
GitHubCommit commit = commits[i].FirstOrDefault();
return commit != null
? new RepositoryContentWithCommitInfo(file, commit, null, commit.Commit.Committer.Date.DateTime)
: new RepositoryContentWithCommitInfo(file);
}));
}
System.Diagnostics.Debug.WriteLine("[DEBUG] HTML parsing");
/* ================
* HTML STRUCTURE
* ================
* ...
* <tr class="js-navigation-item">
* ...
* <td class="content">
* <span ...>
* <a href="CONTENT_URL">...</a>
* </span>
* </td>
* <td class="message">
* <span ...>
* [...]?
* <a title="COMMIT_MESSAGE">...</a>
* </span>
* </td>
* <td class="age">
* <span ...>
* <time-ago datetime="EDIT_TIME">...</a>
* </span>
* </td>
* ... */
// Try to extract the commit info, in parallel
int cores = Environment.ProcessorCount;
List <RepositoryContentWithCommitInfo>[] partials =
(from i in Enumerable.Range(1, cores)
let list = new List <RepositoryContentWithCommitInfo>()
select list).ToArray();
ParallelLoopResult result = Parallel.For(0, cores, new ParallelOptions {
MaxDegreeOfParallelism = cores
}, workerId =>
{
int max = contents.Count * (workerId + 1) / cores;
for (int i = contents.Count * workerId / cores; i < max; i++)
{
// Find the right node
RepositoryContent element = contents[i];
HtmlNode target =
document.DocumentNode?.Descendants("a")
?.FirstOrDefault(child => child.Attributes?.AttributesWithName("id")
?.FirstOrDefault()?.Value?.EndsWith(element.Sha) == true);
// Parse the node contents
if (target != null)
{
// Get the commit and time nodes
HtmlNode
messageRoot = target.Ancestors("td")?.FirstOrDefault()?.Siblings()?.FirstOrDefault(node => node.Name.Equals("td")),
timeRoot = messageRoot?.Siblings()?.FirstOrDefault(node => node.Name.Equals("td"));
HtmlAttribute
messageTitle = messageRoot?.Descendants("a")
?.Select(node => node.Attributes?.AttributesWithName("title")?.FirstOrDefault())
?.FirstOrDefault(node => node != null),
timestamp = timeRoot?.Descendants("time-ago")?.FirstOrDefault()?.Attributes?.AttributesWithName("datetime")?.FirstOrDefault();
// Fix the message, if present
String message = messageTitle?.Value;
if (message != null)
{
message = WebUtility.HtmlDecode(message); // Remove HTML-encoded characters
message = Regex.Replace(message, @":[^:]+: ?| ?:[^:]+:", String.Empty); // Remove GitHub emojis
}
// Add the parsed contents
if (timestamp?.Value != null)
{
DateTime time;
if (DateTime.TryParse(timestamp.Value, out time))
{
partials[workerId].Add(new RepositoryContentWithCommitInfo(element, null, message, time));
continue;
}
}
partials[workerId].Add(new RepositoryContentWithCommitInfo(element, null, message));
continue;
}
partials[workerId].Add(new RepositoryContentWithCommitInfo(element));
}
});
if (!result.IsCompleted)
{
throw new InvalidOperationException();
}
return(partials.SelectMany(list => list).OrderByDescending(entry => entry.Content.Type));
}
catch
{
// Just return the original content without additional info
return(contents?.OrderByDescending(entry => entry.Type).Select(content => new RepositoryContentWithCommitInfo(content)));
}
}
