private void IssueList_ListChanged(object sender, ListChangedEventArgs e) { var list = IssueList.ToArray(); foreach (var item in list) { // If issue is unchecked or no longer a problem then... if (Warnings.Contains(item) && (!item.IsEnabled || !item.Severity.HasValue || item.Severity == IssueSeverity.None)) { // Remove from warnings list. Warnings.Remove(item); } // If issue not found and problem found then... else if (!Warnings.Contains(item) && item.IsEnabled && item.Severity.HasValue && item.Severity.Value != IssueSeverity.None) { // Add to warnings list. Warnings.Add(item); } } // Get issues in progress. list = IssueList.Where(x => x.Status != IssueStatus.Idle).ToArray(); var sb = new StringBuilder(); foreach (var item in list) { if (sb.Length > 0) { sb.Append(", "); } sb.AppendFormat("{0}/{1} {2}: {3}", IssueList.IndexOf(item), IssueList.Count, item.GetType().Name, item.Status); } StatusLabel.Text = sb.ToString(); UpdateIgnoreAllButton(); UpdateNoIssuesPanel(); }
public static async Task RunAsync(ArgsReader args) { // read command-line arguments var dateString = args.ReadOption("date"); var authTokens = new Queue<string>(); while (args.ReadOption("auth") is string authToken) authTokens.Enqueue(authToken); var isQuiet = args.ReadFlag("quiet"); var isVerbose = args.ReadFlag("verbose") && !isQuiet; var outputDirectory = args.ReadOption("output"); var cacheDirectory = args.ReadOption("cache"); var emailFrom = args.ReadOption("email-from"); var emailTo = args.ReadOption("email-to"); var emailSubject = args.ReadOption("email-subject"); var emailSmtp = args.ReadOption("email-smtp"); var emailUsername = args.ReadOption("email-user"); var emailPassword = args.ReadOption("email-pwd"); var configFilePath = args.ReadArgument(); args.VerifyComplete(); // find config file if (configFilePath is null) throw new ApplicationException(GetUsage()); configFilePath = Path.GetFullPath(configFilePath); if (!File.Exists(configFilePath)) throw new ApplicationException("Configuration file not found."); // deserialize config file var settings = JsonSerializer.Deserialize<DigestSettings>( ConvertYamlToJson(File.ReadAllText(configFilePath)), new JsonSerializerOptions { PropertyNameCaseInsensitive = true }) ?? throw new ApplicationException("Invalid configuration."); // determine date/time range in UTC var timeZoneOffset = settings.TimeZoneOffsetHours is not null ? TimeSpan.FromHours(settings.TimeZoneOffsetHours.Value) : DateTimeOffset.Now.Offset; var now = new DateTimeOffset(DateTime.UtcNow).ToOffset(timeZoneOffset); var todayIso = now.Date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); var date = ParseDateArgument(dateString, now.Date); var dateIso = date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); var startDateTimeUtc = new DateTimeOffset(date.Year, date.Month, date.Day, 0, 0, 0, timeZoneOffset).UtcDateTime; var endDateTimeUtc = startDateTimeUtc.AddDays(1.0); // determine culture var culture = settings.Culture is null ? CultureInfo.CurrentCulture : CultureInfo.GetCultureInfo(settings.Culture); // determine output file or email string? outputFile = null; if (emailTo is not null) { if (outputDirectory is not null) throw new ApplicationException("Cannot use both --email-to and --output."); if (emailFrom is null) throw new ApplicationException("Missing required --email-from."); if (emailSmtp is null) throw new ApplicationException("Missing required --email-smtp."); } else { // determine output file outputDirectory = Path.GetFullPath(outputDirectory ?? "."); outputFile = Path.Combine(outputDirectory, $"{dateIso}.html"); } // get GitHub settings var githubs = new List<GitHubSettings>(); if (settings.GitHub is not null) githubs.Add(settings.GitHub); if (settings.GitHubs is not null) githubs.AddRange(settings.GitHubs); if (githubs.Count == 0) throw new ApplicationException("Configuration file must specify at least one github."); // create report var report = new ReportData { Date = date, PreviousDate = date.AddDays(-1), Now = now, IsEmail = emailTo is not null, }; try { foreach (var github in githubs) { // prepare HTTP client var httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.UserAgent.Add(ProductInfoHeaderValue.Parse("GitHubDigestBuilder")); var authToken = (github.AuthTokenEnv is null ? null : Environment.GetEnvironmentVariable(github.AuthTokenEnv)) ?? github.AuthToken; if (authToken is null) authTokens.TryDequeue(out authToken); if (!string.IsNullOrWhiteSpace(authToken)) httpClient.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse($"token {authToken}"); string webBase; string apiBase; if (github.Enterprise is string enterprise) { webBase = enterprise; if (!webBase.StartsWith("http://", StringComparison.OrdinalIgnoreCase) && !webBase.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) webBase = "https://" + webBase.TrimEnd('/'); apiBase = webBase + "/api/v3"; } else { webBase = "https://github.com"; apiBase = "https://api.github.com"; } const int cacheVersion = 2; using var sha1 = SHA1.Create(); cacheDirectory = cacheDirectory is not null ? Path.GetFullPath(cacheDirectory) : Path.Combine(Path.GetTempPath(), "GitHubDigestBuilderCache"); cacheDirectory = Path.Combine(cacheDirectory, BitConverter.ToString(sha1.ComputeHash(Encoding.UTF8.GetBytes($"{cacheVersion} {apiBase} {authToken}"))).Replace("-", "")[..16]); Directory.CreateDirectory(cacheDirectory); // don't process the same event twice var handledEventIds = new HashSet<string>(); // stop when rate limited var rateLimitResetUtc = default(DateTime?); async Task<PagedDownloadResult> LoadPagesAsync(string url, string[] accepts, int maxPageCount, Func<JsonElement, bool>? isLastPage = null) { var cacheName = Regex.Replace(Regex.Replace(url, @"\?.*$", ""), @"/+", "_").Trim('_'); var cacheFile = Path.Combine(cacheDirectory, $"{cacheName}.json"); var cacheElement = default(JsonElement); string? etag = null; if (File.Exists(cacheFile)) { await using var cacheReadStream = File.OpenRead(cacheFile); cacheElement = (await JsonDocument.ParseAsync(cacheReadStream)).RootElement; // if we're asking for the same date, and the cache was generated for a previous day, assume it is still good var cacheDateIso = cacheElement.GetProperty("date").GetString(); var cacheTodayIso = cacheElement.TryGetProperty("today")?.GetString(); if (cacheDateIso == dateIso && string.CompareOrdinal(cacheDateIso, cacheTodayIso) < 0) { return new PagedDownloadResult( Enum.Parse<DownloadStatus>(cacheElement.GetProperty("status").GetString() ?? throw new InvalidOperationException("Missing status.")), cacheElement.GetProperty("items").EnumerateArray().ToList()); } // don't use the cache if we may have stopped early and we're asking for an older report if (isLastPage is null || string.CompareOrdinal(dateIso, cacheDateIso) >= 0) etag = cacheElement.GetProperty("etag").GetString(); } if (rateLimitResetUtc is object) return new PagedDownloadResult(DownloadStatus.RateLimited); var status = DownloadStatus.TooMuchActivity; var items = new List<JsonElement>(); for (var pageNumber = 1; pageNumber <= maxPageCount; pageNumber++) { var pageParameter = pageNumber == 1 ? "" : $"{(url.Contains('?') ? '&' : '?')}page={pageNumber}"; var request = new HttpRequestMessage(HttpMethod.Get, $"{apiBase}/{url}{pageParameter}"); foreach (var accept in accepts) request.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse(accept)); if (pageNumber == 1 && etag is not null) request.Headers.IfNoneMatch.Add(EntityTagHeaderValue.Parse(etag)); var response = await httpClient!.SendAsync(request); if (isVerbose) Console.WriteLine($"{request.RequestUri!.AbsoluteUri} [{response.StatusCode}]"); if (pageNumber == 1 && etag is not null && response.StatusCode == HttpStatusCode.NotModified) { status = Enum.Parse<DownloadStatus>(cacheElement.GetProperty("status").GetString() ?? throw new InvalidOperationException("Missing status.")); items.AddRange(cacheElement.GetProperty("items").EnumerateArray()); break; } if (response.StatusCode == HttpStatusCode.NotFound) return new PagedDownloadResult(DownloadStatus.NotFound); if (response.StatusCode == HttpStatusCode.Forbidden && response.Headers.TryGetValues("X-RateLimit-Remaining", out var rateLimitRemainingValues) && rateLimitRemainingValues.FirstOrDefault() == "0" && response.Headers.TryGetValues("X-RateLimit-Reset", out var rateLimitResetValues) && int.TryParse(rateLimitResetValues.FirstOrDefault() ?? "", out var resetEpochSeconds)) { rateLimitResetUtc = DateTime.UnixEpoch.AddSeconds(resetEpochSeconds); var message = "GitHub API rate limit exceeded. " + $"Try again at {new DateTimeOffset(rateLimitResetUtc.Value).ToOffset(timeZoneOffset).ToString("f", culture)}."; if (authToken is null) message += " Specify a GitHub personal access token for a much higher rate limit."; AddWarning(message); return new PagedDownloadResult(DownloadStatus.RateLimited); } if (response.StatusCode == HttpStatusCode.Unauthorized) throw new ApplicationException("GitHub API returned 401 Unauthorized. Ensure that your auth token is set to a valid personal access token."); if (response.StatusCode != HttpStatusCode.OK) throw new InvalidOperationException($"Unexpected status code: {response.StatusCode}"); if (pageNumber == 1) etag = (response.Headers.ETag ?? throw new InvalidOperationException("Missing ETag.")).Tag; await using var pageStream = await response.Content.ReadAsStreamAsync(); var pageDocument = await JsonDocument.ParseAsync(pageStream); items.AddRange(pageDocument.RootElement.EnumerateArray()); if (pageDocument.RootElement.GetArrayLength() == 0 || isLastPage?.Invoke(pageDocument.RootElement) == true) { status = DownloadStatus.Success; break; } } await using var cacheWriteStream = File.Open(cacheFile, FileMode.Create, FileAccess.Write); await JsonSerializer.SerializeAsync(cacheWriteStream, new { status = status.ToString(), etag, date = dateIso, today = todayIso, items, }, new JsonSerializerOptions { WriteIndented = true }); return new PagedDownloadResult(status, items); } void AddWarning(string text) { if (!report!.Warnings.Contains(text)) { report.Warnings.Add(text); Console.Error.WriteLine(text); } } var sourceRepoNames = new List<string>(); var sourceRepoIndices = new Dictionary<string, int>(); void AddRepoForSource(string repoName, int sourceIndex) { if (!sourceRepoIndices.ContainsKey(repoName)) { sourceRepoNames.Add(repoName); sourceRepoIndices.Add(repoName, sourceIndex); } } async Task AddReposForSource(string sourceKind, string sourceName, int sourceIndex, string? topic) { var orgRepoNames = new HashSet<string>(); var result = await LoadPagesAsync($"{sourceKind}/{sourceName}/repos?sort=updated&per_page=100", accepts: new[] { "application/vnd.github.mercy-preview+json" }, maxPageCount: 100); foreach (var repoElement in result.Elements) { if (!repoElement.GetProperty("private").GetBoolean() && !repoElement.GetProperty("archived").GetBoolean() && !repoElement.GetProperty("disabled").GetBoolean() && (topic is null || repoElement.GetProperty("topics").EnumerateArray().Select(x => x.GetString()).Any(x => x == topic))) { orgRepoNames.Add(repoElement.GetProperty("full_name").GetString() ?? throw new InvalidOperationException("Missing full_name.")); } } foreach (var orgRepoName in orgRepoNames) AddRepoForSource(orgRepoName, sourceIndex); if (result.Status == DownloadStatus.TooMuchActivity) AddWarning($"Too many updated repositories found for {sourceName}."); else if (result.Status == DownloadStatus.NotFound) AddWarning($"Failed to find repositories for {sourceName}."); } var settingsRepos = github.Repos ?? new List<RepoSettings>(); var settingsUsers = github.Users ?? new List<UserSettings>(); if (settingsRepos.Count == 0 && settingsUsers.Count == 0) throw new ApplicationException("No repositories or users specified in configuration."); foreach (var (settingsRepo, sourceIndex) in settingsRepos.Select((x, i) => (x, i))) { switch (settingsRepo) { case { Name: string name, User: null, Org: null, Topic: null }: AddRepoForSource(name, sourceIndex); break; case { Name: null, User: string user, Org: null }: