private void IssueList_ListChanged(object sender, ListChangedEventArgs e)
{
var list = IssueList.ToArray();
foreach (var item in list)
{
// If issue is unchecked or no longer a problem then...
if (Warnings.Contains(item) && (!item.IsEnabled || !item.Severity.HasValue || item.Severity == IssueSeverity.None))
{
// Remove from warnings list.
Warnings.Remove(item);
}
// If issue not found and problem found then...
else if (!Warnings.Contains(item) && item.IsEnabled && item.Severity.HasValue && item.Severity.Value != IssueSeverity.None)
{
// Add to warnings list.
Warnings.Add(item);
}
}
// Get issues in progress.
list = IssueList.Where(x => x.Status != IssueStatus.Idle).ToArray();
var sb = new StringBuilder();
foreach (var item in list)
{
if (sb.Length > 0)
{
sb.Append(", ");
}
sb.AppendFormat("{0}/{1} {2}: {3}", IssueList.IndexOf(item), IssueList.Count, item.GetType().Name, item.Status);
}
StatusLabel.Text = sb.ToString();
UpdateIgnoreAllButton();
UpdateNoIssuesPanel();
}
public static async Task RunAsync(ArgsReader args)
{
// read command-line arguments
var dateString = args.ReadOption("date");
var authTokens = new Queue<string>();
while (args.ReadOption("auth") is string authToken)
authTokens.Enqueue(authToken);
var isQuiet = args.ReadFlag("quiet");
var isVerbose = args.ReadFlag("verbose") && !isQuiet;
var outputDirectory = args.ReadOption("output");
var cacheDirectory = args.ReadOption("cache");
var emailFrom = args.ReadOption("email-from");
var emailTo = args.ReadOption("email-to");
var emailSubject = args.ReadOption("email-subject");
var emailSmtp = args.ReadOption("email-smtp");
var emailUsername = args.ReadOption("email-user");
var emailPassword = args.ReadOption("email-pwd");
var configFilePath = args.ReadArgument();
args.VerifyComplete();
// find config file
if (configFilePath is null)
throw new ApplicationException(GetUsage());
configFilePath = Path.GetFullPath(configFilePath);
if (!File.Exists(configFilePath))
throw new ApplicationException("Configuration file not found.");
// deserialize config file
var settings = JsonSerializer.Deserialize<DigestSettings>(
ConvertYamlToJson(File.ReadAllText(configFilePath)),
new JsonSerializerOptions { PropertyNameCaseInsensitive = true }) ?? throw new ApplicationException("Invalid configuration.");
// determine date/time range in UTC
var timeZoneOffset = settings.TimeZoneOffsetHours is not null ? TimeSpan.FromHours(settings.TimeZoneOffsetHours.Value) : DateTimeOffset.Now.Offset;
var now = new DateTimeOffset(DateTime.UtcNow).ToOffset(timeZoneOffset);
var todayIso = now.Date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
var date = ParseDateArgument(dateString, now.Date);
var dateIso = date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
var startDateTimeUtc = new DateTimeOffset(date.Year, date.Month, date.Day, 0, 0, 0, timeZoneOffset).UtcDateTime;
var endDateTimeUtc = startDateTimeUtc.AddDays(1.0);
// determine culture
var culture = settings.Culture is null ? CultureInfo.CurrentCulture : CultureInfo.GetCultureInfo(settings.Culture);
// determine output file or email
string? outputFile = null;
if (emailTo is not null)
{
if (outputDirectory is not null)
throw new ApplicationException("Cannot use both --email-to and --output.");
if (emailFrom is null)
throw new ApplicationException("Missing required --email-from.");
if (emailSmtp is null)
throw new ApplicationException("Missing required --email-smtp.");
}
else
{
// determine output file
outputDirectory = Path.GetFullPath(outputDirectory ?? ".");
outputFile = Path.Combine(outputDirectory, $"{dateIso}.html");
}
// get GitHub settings
var githubs = new List<GitHubSettings>();
if (settings.GitHub is not null)
githubs.Add(settings.GitHub);
if (settings.GitHubs is not null)
githubs.AddRange(settings.GitHubs);
if (githubs.Count == 0)
throw new ApplicationException("Configuration file must specify at least one github.");
// create report
var report = new ReportData
{
Date = date,
PreviousDate = date.AddDays(-1),
Now = now,
IsEmail = emailTo is not null,
};
try
{
foreach (var github in githubs)
{
// prepare HTTP client
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.UserAgent.Add(ProductInfoHeaderValue.Parse("GitHubDigestBuilder"));
var authToken = (github.AuthTokenEnv is null ? null : Environment.GetEnvironmentVariable(github.AuthTokenEnv)) ?? github.AuthToken;
if (authToken is null)
authTokens.TryDequeue(out authToken);
if (!string.IsNullOrWhiteSpace(authToken))
httpClient.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse($"token {authToken}");
string webBase;
string apiBase;
if (github.Enterprise is string enterprise)
{
webBase = enterprise;
if (!webBase.StartsWith("http://", StringComparison.OrdinalIgnoreCase) && !webBase.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
webBase = "https://" + webBase.TrimEnd('/');
apiBase = webBase + "/api/v3";
}
else
{
webBase = "https://github.com";
apiBase = "https://api.github.com";
}
const int cacheVersion = 2;
using var sha1 = SHA1.Create();
cacheDirectory = cacheDirectory is not null ? Path.GetFullPath(cacheDirectory) : Path.Combine(Path.GetTempPath(), "GitHubDigestBuilderCache");
cacheDirectory = Path.Combine(cacheDirectory, BitConverter.ToString(sha1.ComputeHash(Encoding.UTF8.GetBytes($"{cacheVersion} {apiBase} {authToken}"))).Replace("-", "")[..16]);
Directory.CreateDirectory(cacheDirectory);
// don't process the same event twice
var handledEventIds = new HashSet<string>();
// stop when rate limited
var rateLimitResetUtc = default(DateTime?);
async Task<PagedDownloadResult> LoadPagesAsync(string url, string[] accepts, int maxPageCount, Func<JsonElement, bool>? isLastPage = null)
{
var cacheName = Regex.Replace(Regex.Replace(url, @"\?.*$", ""), @"/+", "_").Trim('_');
var cacheFile = Path.Combine(cacheDirectory, $"{cacheName}.json");
var cacheElement = default(JsonElement);
string? etag = null;
if (File.Exists(cacheFile))
{
await using var cacheReadStream = File.OpenRead(cacheFile);
cacheElement = (await JsonDocument.ParseAsync(cacheReadStream)).RootElement;
// if we're asking for the same date, and the cache was generated for a previous day, assume it is still good
var cacheDateIso = cacheElement.GetProperty("date").GetString();
var cacheTodayIso = cacheElement.TryGetProperty("today")?.GetString();
if (cacheDateIso == dateIso && string.CompareOrdinal(cacheDateIso, cacheTodayIso) < 0)
{
return new PagedDownloadResult(
Enum.Parse<DownloadStatus>(cacheElement.GetProperty("status").GetString() ?? throw new InvalidOperationException("Missing status.")),
cacheElement.GetProperty("items").EnumerateArray().ToList());
}
// don't use the cache if we may have stopped early and we're asking for an older report
if (isLastPage is null || string.CompareOrdinal(dateIso, cacheDateIso) >= 0)
etag = cacheElement.GetProperty("etag").GetString();
}
if (rateLimitResetUtc is object)
return new PagedDownloadResult(DownloadStatus.RateLimited);
var status = DownloadStatus.TooMuchActivity;
var items = new List<JsonElement>();
for (var pageNumber = 1; pageNumber <= maxPageCount; pageNumber++)
{
var pageParameter = pageNumber == 1 ? "" : $"{(url.Contains('?') ? '&' : '?')}page={pageNumber}";
var request = new HttpRequestMessage(HttpMethod.Get, $"{apiBase}/{url}{pageParameter}");
foreach (var accept in accepts)
request.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse(accept));
if (pageNumber == 1 && etag is not null)
request.Headers.IfNoneMatch.Add(EntityTagHeaderValue.Parse(etag));
var response = await httpClient!.SendAsync(request);
if (isVerbose)
Console.WriteLine($"{request.RequestUri!.AbsoluteUri} [{response.StatusCode}]");
if (pageNumber == 1 && etag is not null && response.StatusCode == HttpStatusCode.NotModified)
{
status = Enum.Parse<DownloadStatus>(cacheElement.GetProperty("status").GetString() ?? throw new InvalidOperationException("Missing status."));
items.AddRange(cacheElement.GetProperty("items").EnumerateArray());
break;
}
if (response.StatusCode == HttpStatusCode.NotFound)
return new PagedDownloadResult(DownloadStatus.NotFound);
if (response.StatusCode == HttpStatusCode.Forbidden &&
response.Headers.TryGetValues("X-RateLimit-Remaining", out var rateLimitRemainingValues) &&
rateLimitRemainingValues.FirstOrDefault() == "0" &&
response.Headers.TryGetValues("X-RateLimit-Reset", out var rateLimitResetValues) &&
int.TryParse(rateLimitResetValues.FirstOrDefault() ?? "", out var resetEpochSeconds))
{
rateLimitResetUtc = DateTime.UnixEpoch.AddSeconds(resetEpochSeconds);
var message = "GitHub API rate limit exceeded. " +
$"Try again at {new DateTimeOffset(rateLimitResetUtc.Value).ToOffset(timeZoneOffset).ToString("f", culture)}.";
if (authToken is null)
message += " Specify a GitHub personal access token for a much higher rate limit.";
AddWarning(message);
return new PagedDownloadResult(DownloadStatus.RateLimited);
}
if (response.StatusCode == HttpStatusCode.Unauthorized)
throw new ApplicationException("GitHub API returned 401 Unauthorized. Ensure that your auth token is set to a valid personal access token.");
if (response.StatusCode != HttpStatusCode.OK)
throw new InvalidOperationException($"Unexpected status code: {response.StatusCode}");
if (pageNumber == 1)
etag = (response.Headers.ETag ?? throw new InvalidOperationException("Missing ETag.")).Tag;
await using var pageStream = await response.Content.ReadAsStreamAsync();
var pageDocument = await JsonDocument.ParseAsync(pageStream);
items.AddRange(pageDocument.RootElement.EnumerateArray());
if (pageDocument.RootElement.GetArrayLength() == 0 || isLastPage?.Invoke(pageDocument.RootElement) == true)
{
status = DownloadStatus.Success;
break;
}
}
await using var cacheWriteStream = File.Open(cacheFile, FileMode.Create, FileAccess.Write);
await JsonSerializer.SerializeAsync(cacheWriteStream, new
{
status = status.ToString(),
etag,
date = dateIso,
today = todayIso,
items,
}, new JsonSerializerOptions { WriteIndented = true });
return new PagedDownloadResult(status, items);
}
void AddWarning(string text)
{
if (!report!.Warnings.Contains(text))
{
report.Warnings.Add(text);
Console.Error.WriteLine(text);
}
}
var sourceRepoNames = new List<string>();
var sourceRepoIndices = new Dictionary<string, int>();
void AddRepoForSource(string repoName, int sourceIndex)
{
if (!sourceRepoIndices.ContainsKey(repoName))
{
sourceRepoNames.Add(repoName);
sourceRepoIndices.Add(repoName, sourceIndex);
}
}
async Task AddReposForSource(string sourceKind, string sourceName, int sourceIndex, string? topic)
{
var orgRepoNames = new HashSet<string>();
var result = await LoadPagesAsync($"{sourceKind}/{sourceName}/repos?sort=updated&per_page=100",
accepts: new[] { "application/vnd.github.mercy-preview+json" }, maxPageCount: 100);
foreach (var repoElement in result.Elements)
{
if (!repoElement.GetProperty("private").GetBoolean() &&
!repoElement.GetProperty("archived").GetBoolean() &&
!repoElement.GetProperty("disabled").GetBoolean() &&
(topic is null || repoElement.GetProperty("topics").EnumerateArray().Select(x => x.GetString()).Any(x => x == topic)))
{
orgRepoNames.Add(repoElement.GetProperty("full_name").GetString() ?? throw new InvalidOperationException("Missing full_name."));
}
}
foreach (var orgRepoName in orgRepoNames)
AddRepoForSource(orgRepoName, sourceIndex);
if (result.Status == DownloadStatus.TooMuchActivity)
AddWarning($"Too many updated repositories found for {sourceName}.");
else if (result.Status == DownloadStatus.NotFound)
AddWarning($"Failed to find repositories for {sourceName}.");
}
var settingsRepos = github.Repos ?? new List<RepoSettings>();
var settingsUsers = github.Users ?? new List<UserSettings>();
if (settingsRepos.Count == 0 && settingsUsers.Count == 0)
throw new ApplicationException("No repositories or users specified in configuration.");
foreach (var (settingsRepo, sourceIndex) in settingsRepos.Select((x, i) => (x, i)))
{
switch (settingsRepo)
{
case { Name: string name, User: null, Org: null, Topic: null }:
AddRepoForSource(name, sourceIndex);
break;
case { Name: null, User: string user, Org: null }:
