public RepositoryContentWithCommitInfo([NotNull] RepositoryContent content, [CanBeNull] GitHubCommit commit = null, [CanBeNull] string message = null, [CanBeNull] DateTime?editTime = null) { Content = content; Commit = commit; _CommitMessage = message; LastEditTime = editTime; }
private async Task FillInfoByLogin(GitHubCommit commit, Data data) { var author = commit.Author; if (author == null) { var fullName = commit.Commit.Author.Name.Split(" "); var firstName = GetEmptyOrName(0, fullName); var lastName = GetEmptyOrName(1, fullName); data.CommitAuthorFirstName = firstName; data.CommitAuthorLastName = lastName; data.CommitAuthorLogin = ""; data.DevID = GetIntIdByString(data.CommitAuthorEmail + firstName + lastName); } else { var login = author.Login; var user = await GetUser(login); var fullName = user.Name.Split(" "); var firstName = GetEmptyOrName(0, fullName); var lastName = GetEmptyOrName(1, fullName); data.CommitAuthorFirstName = firstName; data.CommitAuthorLastName = lastName; data.CommitAuthorLogin = login; //data.DevID = author.Id; data.DevID = GetIntIdByString(data.CommitAuthorEmail + firstName + lastName); } }
/// <summary> /// Handles creating an Event object from a GitHub commit which will be streamed into Splunk. /// </summary> /// <param name="githubCommit">The individual GithubCommit object which holds the data for a commit.</param> /// <param name="eventWriter">The EventWriter for streaming events to Splunk.</param> /// <param name="owner">The GitHub repository owner's name.</param> /// <param name="repositoryName">The GitHub repository's name.</param> public async Task StreamCommit(GitHubCommit githubCommit, EventWriter eventWriter, string owner, string repositoryName) { string authorName = githubCommit.Commit.Author.Name; DateTime date = githubCommit.Commit.Author.Date; // Replace any newlines with a space string commitMessage = Regex.Replace(githubCommit.Commit.Message, "\\n|\\r", " "); dynamic json = new JObject(); json.sha = githubCommit.Sha; json.api_url = githubCommit.Url; json.url = "http://github.com/" + owner + "/" + repositoryName + "/commit/" + githubCommit.Sha; json.message = commitMessage; json.author = authorName; json.date = date.ToString(); var commitEvent = new Event(); commitEvent.Stanza = repositoryName; commitEvent.SourceType = "github_commits"; commitEvent.Time = date; commitEvent.Data = json.ToString(Formatting.None); await eventWriter.QueueEventForWriting(commitEvent); }
public static async Task <string> GetUpdateText(string path) { GitHubClient client = GetClient(path); UpdaterSettings options = UpdaterSettings.Load(path); IReadOnlyList <Release> releases = await client.Repository.Release.GetAll(Settings.Default.RepositoryOwner, Settings.Default.RepositoryName); IReadOnlyList <GitHubCommit> commits = await client.Repository.Commit.GetAll(Settings.Default.RepositoryOwner, Settings.Default.RepositoryName); IEnumerable <Release> latestReleases = releases.OrderByDescending(c => c.CreatedAt).Take(15); StringBuilder commitMessage = new StringBuilder(); foreach (Release release in latestReleases.Where(e => options.InstallPrereleases || !e.Prerelease)) { string releaseMessage = release.Body; if (string.IsNullOrEmpty(releaseMessage)) { GitHubCommit commit = commits.FirstOrDefault(c => c.Commit.Sha == release.TargetCommitish); releaseMessage = commit?.Commit.Message ?? "Unknown"; } commitMessage.AppendLine($"{release.CreatedAt.Date.Date.ToShortDateString()}:"); commitMessage.AppendLine(releaseMessage); commitMessage.AppendLine(); } return(commitMessage.ToString()); }
public async Task <int> UpdateGitHubActionPullRequestCommits(string clientId, string clientSecret, TableStorageAuth tableStorageAuth, string owner, string repo, string pull_number) { GitHubAPIAccess api = new GitHubAPIAccess(); JArray items = await api.GetGitHubPullRequestCommitsJArray(clientId, clientSecret, owner, repo, pull_number); int itemsAdded = 0; TableStorageCommonDA tableDA = new TableStorageCommonDA(tableStorageAuth, tableStorageAuth.TableGitHubPRCommits); //Check each build to see if it's in storage, adding the items not in storage foreach (JToken item in items) { GitHubCommit commit = JsonConvert.DeserializeObject <GitHubCommit>(item.ToString()); string partitionKey = CreateGitHubPRCommitPartitionKey(owner, repo, pull_number); string rowKey = commit.sha; AzureStorageTableModel newItem = new AzureStorageTableModel(partitionKey, rowKey, item.ToString()); if (await tableDA.AddItem(newItem) == true) { itemsAdded++; } } return(itemsAdded); }
public static async Task <string> GetCommitAuthorAsync( string repositoryUrl, string commit, string personalAccessToken) { (string owner, string repoName) = ParseRepoUri(repositoryUrl); Octokit.GitHubClient client = new Octokit.GitHubClient(new ProductHeaderValue("assets-publisher")); Credentials tokenAuth = new Credentials(personalAccessToken); client.Credentials = tokenAuth; GitHubCommit commitInfo = await client.Repository.Commit.Get(owner, repoName, commit); while (commitInfo.Author.Type == "Bot") { if (!commitInfo.Parents.Any()) { break; } commit = commitInfo.Parents.First().Sha; commitInfo = await client.Repository.Commit.Get(owner, repoName, commit); } return($"@{commitInfo.Author.Login}"); }
private GitHubCommentModel CreateCommentModel(int version, GitHubComment comment) { // create URL for the commit from the comment URL string commentUrl = comment.html_url.AbsoluteUri; int hashIndex = commentUrl.IndexOf('#'); string commitUrl = hashIndex == -1 ? commentUrl : commentUrl.Substring(0, hashIndex); // create basic model GitHubCommentModel model = new GitHubCommentModel { CommentUrl = commentUrl, CommitUrl = commitUrl, CommentBody = new HtmlString(comment.body_html), CommitId = comment.commit_id.Substring(0, 8), FilePath = comment.path, LineNumber = comment.line.GetValueOrDefault() == 0 ? null : comment.line, Commenter = comment.user.login, }; // add extra details if present if (version == 2) { GitHubCommit commit = m_commits[comment.commit_id]; string author = commit.author != null ? commit.author.login : "******"; model.Author = author; model.CommitMessage = commit.commit.message; model.CommitFiles = commit.files.Select(f => f.filename).OrderBy(f => f, StringComparer.OrdinalIgnoreCase).ToList(); } return(model); }
private bool TryMemeCommitMessage(GitHubCommit commit, ChannelState chnl) { List<string> topLineList = commit.Commit.Message.Split(' ').ToList<string>(); List<string> bottomLineList = new List<string>(); string topLineString = commit.Commit.Message; string bottomLineString = ""; Channel channel = _client.GetChannel(chnl.ChannelID); while ((topLineString.Length - bottomLineString.Length) > 7 && (topLineString.Count() > 1)) { bottomLineList.Add(topLineList.Last()); topLineList.Remove(topLineList.Last()); bottomLineString = string.Join(" ",bottomLineList.ToArray()); topLineString = string.Join(" ", topLineList.ToArray()); if ((topLineString.Length - bottomLineString.Length) <= 7) { bottomLineList.Reverse(); bottomLineString = string.Join(" ",bottomLineList.ToArray()); Image imageWithMemeText = MemeGeneratingModule.PlaceImageText(topLineString, bottomLineString, "github"); string fileName = "Memes/Memmit" + DateTime.Now.ToString("hhmmss") + ".png"; imageWithMemeText.Save(fileName, ImageFormat.Png); channel.SendFile(fileName); System.Threading.Thread.Sleep(5000); return true; } } return false; }
public static string ToByStr(this GitHubCommit commit) { if (commit.Author != null) { return(commit.Author.ToStr("by")); } return(commit.Commit.Author != null?commit.Commit.Author.ToStr("by") : ""); }
public CommitViewModel Init(string repositoryOwner, string repositoryName, string node, GitHubCommit commit = null) { RepositoryOwner = repositoryOwner; RepositoryName = repositoryName; Commit = commit; Node = node; return(this); }
private string GetCommitInfo(GitHubCommit commit) { var commitAlias = commit.Commits.Count > 1 ? "commits" : "commit"; return($"**{commit.Commits.Count} new {commitAlias}** pushed to " + $"[{commit.Branch}]({commit.CommitUrl})"); }
public CommitViewModel Init(string repositoryOwner, string repositoryName, string node, GitHubCommit commit = null, bool showRepository = false) { RepositoryOwner = repositoryOwner; RepositoryName = repositoryName; Commit = commit; Node = node; ShowRepository = showRepository; return(this); }
private async Task <HttpStatusCode> PushToSpace(GitHubCommit commit) { if (commit == null) { return(HttpStatusCode.InternalServerError); } return(await _spaceService.PushGitHubNotification(commit)); }
public async Task <HttpStatusCode> PushGitHubNotification(GitHubCommit commit) { var url = _urlBuilder.BuildChannelTextMessageUrl("2PTicn3FhRAU"); var content = GetContent(commit); using (var client = _httpClientFactory.CreateClient("spaceHttpClient")) { var result = await client.PostAsync(url, content); return(result.StatusCode); } }
public static GitHubCommit GetNewestCommit() { using (WebClient client = new WebClient()) { client.Headers.Add("User-Agent", "XIVMon"); var result = client.DownloadString($"https://api.github.com/repos/{Repo}/commits"); return(GitHubCommit.FromJson(result)[0]); } }
public GhCommit(GitHubCommit commit) { Message = commit.Commit.Message; if (commit.Author != null) { Author = commit.Author.Id; } if (commit.Files != null) { Files = commit.Files.Select(x => new CommitFile(x)).ToArray(); } }
/** * Check if file is binary or not * return bool */ public async Task <bool> isFileBinary(string[] fileData, GitHubCommit commit) { IReadOnlyList <RepositoryContent> contents; try { // Check if file is binary contents = await client.Repository.Content .GetAllContentsByRef(UserName, RepoName, fileData[0], commit.Sha).ConfigureAwait(false); var targetFile = contents[0]; // Download file string con; using (var wc = new WebClient()) con = wc.DownloadString(targetFile.DownloadUrl); // Check if file is binary for (int i = 1; i < 512 && i < con.Length; i++) { if (con[i] == 0x00 && con[i - 1] == 0x00) { return(true); } } } catch { // Chck if file is binary for deleted or renamed files contents = await client.Repository.Content .GetAllContentsByRef(UserName, RepoName, fileData[0], fileData[1]).ConfigureAwait(false); var targetFile = contents[0]; string con; using (var wc = new WebClient()) con = wc.DownloadString(targetFile.DownloadUrl); // Check if file is binary for (int i = 1; i < 512 && i < con.Length; i++) { if (con[i] == 0x00 && con[i - 1] == 0x00) { return(true); } } } return(false); }
public void UpdateFiles(GitHubCommit commit, DirectoryInfo temp_folder) { if (temp_folder.GetDirectories().Length == 0) { UpdateFiles(commit.Sha, temp_folder); } else { UpdateFiles(commit.Sha, temp_folder); DirectoryInfo[] folders = temp_folder.GetDirectories(); for (int i = 0; i < folders.Length; i++) { UpdateFiles(commit, folders[i]); } } }
private Release ExtractCommits(Repository repository, string currentTag, string previousTag, Release rel, string testPattern, List <Contributor> users) { var @base = previousTag; var head = currentTag; var commits = new List <GitHubCommit>(); var response = repository.Client.Repository.Commit.Compare(repository.Owner, repository.Name, @base, head).Result; rel.NumberofPullRequests = response.Commits.Count; var allIds = new List <int>(); Parallel.ForEach(response.Commits, (c) => { GitHubCommit cmt = null; try { cmt = repository.Client.Repository.Commit.Get(repository.Owner, repository.Name, c.Sha).Result; Console.WriteLine("Commit Id: " + cmt.NodeId); } catch (Exception ex) { Console.WriteLine("Unable to fetch commit : " + ex.Message); } if (cmt != null) { rel.NumberOfAdditions += cmt.Stats.Additions; rel.NumberOfDeletions += cmt.Stats.Deletions; rel.NumberOfFilesModified += cmt.Files.Count; rel.TestFilesChanged += cmt.Files.Count(f => f.BlobUrl.ToLowerInvariant().Contains(testPattern)); rel.CommentsCount += cmt.Commit.CommentCount; if (cmt.Author != null) { allIds.Add(cmt.Author.Id); rel.PriorContributions += users.FirstOrDefault(u => u.Author.Id == cmt.Author.Id) != null ? users.FirstOrDefault(u => u.Author.Id == cmt.Author.Id).Total : 0; // rel.TotalFollowers += client.User. et(cmt.Author.Login).Result.Followers; } } }); rel.NumberOfUniqueContributers = allIds.Distinct().Count(); rel.TestCoverageScore = Math.Abs(rel.NumberOfFilesModified - rel.TestFilesChanged); return(rel); }
private static void ValidateLocalRepository(GitHubCommit expectedCommit) { var root = DirectoryLayout.DetermineRootDirectory(); using (var repo = new LibGit2Sharp.Repository(root)) { string tip = repo.Head.Tip.Sha; if (tip != expectedCommit.Sha) { throw new UserErrorException($"Current local commit: {tip}. Aborting."); } var status = repo.RetrieveStatus("apis/apis.json"); if (status != FileStatus.Unaltered) { throw new UserErrorException($"Expected apis.json to be unaltered. Current status: {status}. Aborting."); } } }
public string BuildChannelCommitMessage(GitHubCommit commit, string channelId) { var template = _messageTemplateBuilder.BuildCommitMessageTemplate(@"JsonTemplates\spaceMessage.json"); template.Recipient.Channel.Id = channelId; template.Content.Sections[0].Elements[0].Content = commit.Pusher.Name; template.Content.Sections[0].Elements[1].Content = GetCommitInfo(commit); template.Content.Sections[0].Elements[2].Content = GetCommits(commit.Commits); return(JsonConvert.SerializeObject( template, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore } )); }
private async Task <Data> GetData(GitHubCommit commit, string owner, string name) { var data = new Data { CommitID = GetIntIdByString(commit.Sha), CommitSha = commit.Sha, CommitAuthorEmail = commit.Commit.Author.Email, CreatedAt = commit.Commit.Author.Date, RepositoryName = owner + "/" + name, }; await FillInfoByLogin(commit, data); var currentCommit = await client.Repository.Commit.Get(owner, name, data.CommitSha); var nameAndSha = currentCommit.Files .Select(x => new NameAndSha { Name = x.Filename, Id = GetIntIdByString(x.Sha) }); data.NameAndSha = nameAndSha; int additions = 0; int deletions = 0; var additionsArrayByCommit = currentCommit.Files.Select(x => x.Additions); var deletionsArrayByCommit = currentCommit.Files.Select(x => x.Deletions); foreach (var a in additionsArrayByCommit) { additions += a; } foreach (var d in deletionsArrayByCommit) { deletions += d; } data.Additions = additions; data.Deletions = deletions; return(data); }
public async void AnnounceCommitMessage(GitHubCommit commit) { foreach (ChannelState chnl in Beta.ChannelStateRepository.ChannelStates) { if (commit.Commit.Message.Length < 110 && chnl.ChatBattleEnabled) { if (!TryMemeCommitMessage(commit, chnl)) return; } else { string msg = "Looks like a new commit was added!\n"; msg += "``` " + commit.Commit.Message + "```"; if (chnl.ChatBattleEnabled) { MessageQueue.Add(new QueuedMessage(msg, chnl.ChannelID)); } } } }
public static GitHubCommitResponse ParseResponse(SocialHttpResponse response) { // Parse the raw JSON response JsonObject obj = response.GetBodyAsJsonObject(); // Check for any errors if (response.StatusCode != HttpStatusCode.OK) { string message = obj.GetString("message"); string url = obj.GetString("documentation_url"); throw new GitHubHttpException(response.StatusCode, message, url); } // Initialize the object to be returned return(new GitHubCommitResponse(response) { Data = GitHubCommit.Parse(obj) }); }
public static async Task <Embed> GetLatestBuild() { GitHubClient github = new GitHubClient(new ProductHeaderValue("Vita3KBot")); Release latestRelease = await github.Repository.Release.Get("Vita3k", "Vita3k", "continous"); ReleaseAsset linuxRelease = latestRelease.Assets.Where(release => { return(release.Name.StartsWith("ubuntu-latest")); }).First(); ReleaseAsset macosRelease = latestRelease.Assets.Where(release => { return(release.Name.StartsWith("macos-latest")); }).First(); ReleaseAsset windowsRelease = latestRelease.Assets.Where(release => { return(release.Name.StartsWith("windows-latest")); }).First(); string commit = latestRelease.Body.Substring(latestRelease.Body.IndexOf(":") + 1).Trim(); commit = commit.Substring(0, commit.Length - 1); GitHubCommit REF = await github.Repository.Commit.Get("Vita3k", "Vita3k", commit); Issue prInfo = await GetPRInfo(github, commit); EmbedBuilder LatestBuild = new EmbedBuilder(); if (prInfo != null) { LatestBuild.WithTitle($"PR: #{prInfo.Number} By {prInfo.User.Login}") .WithUrl(prInfo.HtmlUrl); } else { LatestBuild.WithTitle($"Commit: {REF.Sha} By {REF.Commit.Author.Name}") .WithUrl($"https://github.com/vita3k/vita3k/commit/{REF.Sha}"); } LatestBuild.WithDescription($"{REF.Commit.Message}") .WithColor(Color.Orange) .AddField("Windows", $"[{windowsRelease.Name}]({windowsRelease.BrowserDownloadUrl})") .AddField("Linux", $"[{linuxRelease.Name}]({linuxRelease.BrowserDownloadUrl})") .AddField("Mac", $"[{macosRelease.Name}]({macosRelease.BrowserDownloadUrl})"); return(LatestBuild.Build()); }
internal CommitItemViewModel(GitHubCommit commit, Action <CommitItemViewModel> action) { var msg = commit?.Commit?.Message ?? string.Empty; var firstLine = msg.IndexOf("\n", StringComparison.Ordinal); var description = firstLine > 0 ? msg.Substring(0, firstLine) : msg; _description = new Lazy <string>(() => Emojis.FindAndReplace(description)); var time = DateTimeOffset.MinValue; if (commit.Commit.Committer != null) { time = commit.Commit.Committer.Date; } Time = time.UtcDateTime.Humanize(); Name = commit.GenerateCommiterName(); Avatar = new GitHubAvatar(commit.GenerateGravatarUrl()); Commit = commit; GoToCommand = ReactiveCommand.Create().WithSubscription(_ => action(this)); }
private async Task CreateReleasesAsync(List <ApiMetadata> newReleases, GitHubCommit commit) { var originalMessage = commit.Commit.Message; var unwrappedMessage = string.Join("\n", UnwrapLines(originalMessage.Split('\n'))); Console.WriteLine("Creating releases with tags:"); foreach (var api in newReleases) { var tag = $"{api.Id}-{api.Version}"; var gitRelease = new NewRelease(tag) { Prerelease = !api.IsReleaseVersion, Name = $"{api.Id} version {api.Version}", TargetCommitish = commit.Sha, Body = unwrappedMessage }; // We could parallelize, but there's very little point. await _client.Repository.Release.Create(RepositoryOwner, RepositoryName, gitRelease); Console.WriteLine(tag); } }
private async Task <HttpStatusCode> UpdateLastCommitDateAsync(GitHubCommit lastCommit) { using var command = dbConnection.CreateCommand(); command.Parameters.AddWithValue("@id", 0); command.Parameters.AddWithValue("@data", lastCommit.Commit.Author.Date.ToStringWithSeparatorAndZone(CultureInfo.InvariantCulture)); command.Parameters.AddWithValue("@sha", lastCommit.Sha); command.CommandText = "update [lastcommit] set [data] = @data, [sha] = @sha where[id] = @id"; dbConnection.Open(); int rows = await command.ExecuteNonQueryAsync(); if (rows == 1) { dbConnection.Close(); return(HttpStatusCode.OK); } dbConnection.Close(); return(HttpStatusCode.BadRequest); }
public CommitControl(GitHubCommit commit) { Commit = commit; Wrapper = new Panel(); Wrapper.AutoSize = true; Wrapper.Margin = new Padding(0, 0, 0, 15); // create new label instance HeaderLabel = new System.Windows.Forms.Label(); HeaderLabel.AutoSize = true; if (commit.Commit.Message.IndexOf('\n') > -1) { HeaderLabel.Text = commit.Commit.Message.Substring(0, commit.Commit.Message.IndexOf('\n')); } else { HeaderLabel.Text = commit.Commit.Message; } HeaderLabel.Font = new Font(new FontFamily(System.Drawing.Text.GenericFontFamilies.Serif), 12, FontStyle.Bold); Wrapper.Controls.Add(HeaderLabel); // insertions and deletitions labels InsertionsCountLabel = new System.Windows.Forms.Label(); InsertionsCountLabel.AutoSize = true; InsertionsCountLabel.ForeColor = Color.DarkGreen; InsertionsCountLabel.Text = "+" + commit.Stats.Additions.ToString(); InsertionsCountLabel.Location = new Point(0, HeaderLabel.Height); Wrapper.Controls.Add(InsertionsCountLabel); DeletionsCountLabel = new System.Windows.Forms.Label(); DeletionsCountLabel.AutoSize = true; DeletionsCountLabel.ForeColor = Color.DarkRed; DeletionsCountLabel.Text = "-" + commit.Stats.Deletions.ToString(); DeletionsCountLabel.Location = new Point(0, HeaderLabel.Height + InsertionsCountLabel.Height); Wrapper.Controls.Add(DeletionsCountLabel); }
public static async Task <IEnumerable <RepositoryContentWithCommitInfo> > TryLoadLinkedCommitDataAsync( [NotNull] Task <IReadOnlyList <RepositoryContent> > contentTask, [NotNull] String htmlUrl, [NotNull] GitHubClient client, long repoId, [NotNull] String branch, CancellationToken token) { // Try to download the file info IReadOnlyList <RepositoryContent> contents = null; try { // Load the full HTML body WebView view = new WebView(); TaskCompletionSource <String> tcs = new TaskCompletionSource <String>(); view.NavigationCompleted += (s, e) => { view.InvokeScriptAsync("eval", new[] { "document.documentElement.outerHTML;" }).AsTask().ContinueWith(t => { tcs.SetResult(t.Status == TaskStatus.RanToCompletion ? t.Result : null); }); }; // Manually set the user agent to get the full desktop site String userAgent = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; ARM; Trident/7.0; Touch; rv:11.0; WPDesktop) like Gecko"; HttpRequestMessage httpRequestMessage = new HttpRequestMessage(HttpMethod.Post, new Uri(htmlUrl)); httpRequestMessage.Headers.Append("User-Agent", userAgent); view.NavigateWithHttpRequestMessage(httpRequestMessage); // Run the web calls in parallel await Task.WhenAll(contentTask, tcs.Task); contents = contentTask.Result; String html = tcs.Task.Result; if (token.IsCancellationRequested) { return(contents?.OrderByDescending(entry => entry.Type).Select(content => new RepositoryContentWithCommitInfo(content))); } // Load the HTML document HtmlDocument document = new HtmlDocument(); document.LoadHtml(html); /* ================= * HTML error tags * ================= * ... * <include-fragment class="commit-tease commit-loader"> * ... * <div class="loader-error"/> * </include-fragment> * ... */ // Check if the HTML loading was successful if (document.DocumentNode ?.Descendants("include-fragment") // Get the <include-fragment/> nodes ?.FirstOrDefault(node => node.Attributes?.AttributesWithName("class") // Get the nodes with a class attribute ?.FirstOrDefault(att => att.Value?.Equals("commit-tease commit-loader") == true) // That attribute must have this name != null) // There must be a node with these specs if the HTML loading failed ?.Descendants("div") // Get the inner <div/> nodes ?.FirstOrDefault(node => node.Attributes?.AttributesWithName("class")?.FirstOrDefault() // Check the class name ?.Value?.Equals("loader-error") == true) != null || // Make sure there was in fact a loading error html.Contains("class=\"warning include - fragment - error\"") || html.Contains("Failed to load latest commit information")) { System.Diagnostics.Debug.WriteLine("[DEBUG] Fallback"); // Use the Oktokit APIs to get the info IEnumerable <Task <IReadOnlyList <GitHubCommit> > > tasks = contents.Select(r => client.Repository.Commit.GetAll(repoId, new CommitRequest { Path = r.Path, Sha = branch }, // Only get the commits that edited the current file new ApiOptions { PageCount = 1, PageSize = 1 })); // Just get the latest commit for this file IReadOnlyList <GitHubCommit>[] commits = await Task.WhenAll(tasks); // Query the results return(contents.AsParallel().OrderByDescending(file => file.Type).Select((file, i) => { GitHubCommit commit = commits[i].FirstOrDefault(); return commit != null ? new RepositoryContentWithCommitInfo(file, commit, null, commit.Commit.Committer.Date.DateTime) : new RepositoryContentWithCommitInfo(file); })); } System.Diagnostics.Debug.WriteLine("[DEBUG] HTML parsing"); /* ================ * HTML STRUCTURE * ================ * ... * <tr class="js-navigation-item"> * ... * <td class="content"> * <span ...> * <a href="CONTENT_URL">...</a> * </span> * </td> * <td class="message"> * <span ...> * [...]? * <a title="COMMIT_MESSAGE">...</a> * </span> * </td> * <td class="age"> * <span ...> * <time-ago datetime="EDIT_TIME">...</a> * </span> * </td> * ... */ // Try to extract the commit info, in parallel int cores = Environment.ProcessorCount; List <RepositoryContentWithCommitInfo>[] partials = (from i in Enumerable.Range(1, cores) let list = new List <RepositoryContentWithCommitInfo>() select list).ToArray(); ParallelLoopResult result = Parallel.For(0, cores, new ParallelOptions { MaxDegreeOfParallelism = cores }, workerId => { int max = contents.Count * (workerId + 1) / cores; for (int i = contents.Count * workerId / cores; i < max; i++) { // Find the right node RepositoryContent element = contents[i]; HtmlNode target = document.DocumentNode?.Descendants("a") ?.FirstOrDefault(child => child.Attributes?.AttributesWithName("id") ?.FirstOrDefault()?.Value?.EndsWith(element.Sha) == true); // Parse the node contents if (target != null) { // Get the commit and time nodes HtmlNode messageRoot = target.Ancestors("td")?.FirstOrDefault()?.Siblings()?.FirstOrDefault(node => node.Name.Equals("td")), timeRoot = messageRoot?.Siblings()?.FirstOrDefault(node => node.Name.Equals("td")); HtmlAttribute messageTitle = messageRoot?.Descendants("a") ?.Select(node => node.Attributes?.AttributesWithName("title")?.FirstOrDefault()) ?.FirstOrDefault(node => node != null), timestamp = timeRoot?.Descendants("time-ago")?.FirstOrDefault()?.Attributes?.AttributesWithName("datetime")?.FirstOrDefault(); // Fix the message, if present String message = messageTitle?.Value; if (message != null) { message = WebUtility.HtmlDecode(message); // Remove HTML-encoded characters message = Regex.Replace(message, @":[^:]+: ?| ?:[^:]+:", String.Empty); // Remove GitHub emojis } // Add the parsed contents if (timestamp?.Value != null) { DateTime time; if (DateTime.TryParse(timestamp.Value, out time)) { partials[workerId].Add(new RepositoryContentWithCommitInfo(element, null, message, time)); continue; } } partials[workerId].Add(new RepositoryContentWithCommitInfo(element, null, message)); continue; } partials[workerId].Add(new RepositoryContentWithCommitInfo(element)); } }); if (!result.IsCompleted) { throw new InvalidOperationException(); } return(partials.SelectMany(list => list).OrderByDescending(entry => entry.Content.Type)); } catch { // Just return the original content without additional info return(contents?.OrderByDescending(entry => entry.Type).Select(content => new RepositoryContentWithCommitInfo(content))); } }