Example #1
0
        private void IssueList_ListChanged(object sender, ListChangedEventArgs e)
        {
            var list = IssueList.ToArray();

            foreach (var item in list)
            {
                // If issue is unchecked or no longer a problem then...
                if (Warnings.Contains(item) && (!item.IsEnabled || !item.Severity.HasValue || item.Severity == IssueSeverity.None))
                {
                    // Remove from warnings list.
                    Warnings.Remove(item);
                }
                // If issue not found and problem found then...
                else if (!Warnings.Contains(item) && item.IsEnabled && item.Severity.HasValue && item.Severity.Value != IssueSeverity.None)
                {
                    // Add to warnings list.
                    Warnings.Add(item);
                }
            }
            // Get issues in progress.
            list = IssueList.Where(x => x.Status != IssueStatus.Idle).ToArray();
            var sb = new StringBuilder();

            foreach (var item in list)
            {
                if (sb.Length > 0)
                {
                    sb.Append(", ");
                }
                sb.AppendFormat("{0}/{1} {2}: {3}", IssueList.IndexOf(item), IssueList.Count, item.GetType().Name, item.Status);
            }
            StatusLabel.Text = sb.ToString();
            UpdateIgnoreAllButton();
            UpdateNoIssuesPanel();
        }
Example #2
0
		public static async Task RunAsync(ArgsReader args)
		{
			// read command-line arguments
			var dateString = args.ReadOption("date");
			var authTokens = new Queue<string>();
			while (args.ReadOption("auth") is string authToken)
				authTokens.Enqueue(authToken);
			var isQuiet = args.ReadFlag("quiet");
			var isVerbose = args.ReadFlag("verbose") && !isQuiet;
			var outputDirectory = args.ReadOption("output");
			var cacheDirectory = args.ReadOption("cache");
			var emailFrom = args.ReadOption("email-from");
			var emailTo = args.ReadOption("email-to");
			var emailSubject = args.ReadOption("email-subject");
			var emailSmtp = args.ReadOption("email-smtp");
			var emailUsername = args.ReadOption("email-user");
			var emailPassword = args.ReadOption("email-pwd");
			var configFilePath = args.ReadArgument();
			args.VerifyComplete();

			// find config file
			if (configFilePath is null)
				throw new ApplicationException(GetUsage());
			configFilePath = Path.GetFullPath(configFilePath);
			if (!File.Exists(configFilePath))
				throw new ApplicationException("Configuration file not found.");

			// deserialize config file
			var settings = JsonSerializer.Deserialize<DigestSettings>(
				ConvertYamlToJson(File.ReadAllText(configFilePath)),
				new JsonSerializerOptions { PropertyNameCaseInsensitive = true }) ?? throw new ApplicationException("Invalid configuration.");

			// determine date/time range in UTC
			var timeZoneOffset = settings.TimeZoneOffsetHours is not null ? TimeSpan.FromHours(settings.TimeZoneOffsetHours.Value) : DateTimeOffset.Now.Offset;
			var now = new DateTimeOffset(DateTime.UtcNow).ToOffset(timeZoneOffset);
			var todayIso = now.Date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
			var date = ParseDateArgument(dateString, now.Date);
			var dateIso = date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
			var startDateTimeUtc = new DateTimeOffset(date.Year, date.Month, date.Day, 0, 0, 0, timeZoneOffset).UtcDateTime;
			var endDateTimeUtc = startDateTimeUtc.AddDays(1.0);

			// determine culture
			var culture = settings.Culture is null ? CultureInfo.CurrentCulture : CultureInfo.GetCultureInfo(settings.Culture);

			// determine output file or email
			string? outputFile = null;
			if (emailTo is not null)
			{
				if (outputDirectory is not null)
					throw new ApplicationException("Cannot use both --email-to and --output.");
				if (emailFrom is null)
					throw new ApplicationException("Missing required --email-from.");
				if (emailSmtp is null)
					throw new ApplicationException("Missing required --email-smtp.");
			}
			else
			{
				// determine output file
				outputDirectory = Path.GetFullPath(outputDirectory ?? ".");
				outputFile = Path.Combine(outputDirectory, $"{dateIso}.html");
			}

			// get GitHub settings
			var githubs = new List<GitHubSettings>();
			if (settings.GitHub is not null)
				githubs.Add(settings.GitHub);
			if (settings.GitHubs is not null)
				githubs.AddRange(settings.GitHubs);
			if (githubs.Count == 0)
				throw new ApplicationException("Configuration file must specify at least one github.");

			// create report
			var report = new ReportData
			{
				Date = date,
				PreviousDate = date.AddDays(-1),
				Now = now,
				IsEmail = emailTo is not null,
			};

			try
			{
				foreach (var github in githubs)
				{
					// prepare HTTP client
					var httpClient = new HttpClient();
					httpClient.DefaultRequestHeaders.UserAgent.Add(ProductInfoHeaderValue.Parse("GitHubDigestBuilder"));

					var authToken = (github.AuthTokenEnv is null ? null : Environment.GetEnvironmentVariable(github.AuthTokenEnv)) ?? github.AuthToken;
					if (authToken is null)
						authTokens.TryDequeue(out authToken);
					if (!string.IsNullOrWhiteSpace(authToken))
						httpClient.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse($"token {authToken}");

					string webBase;
					string apiBase;
					if (github.Enterprise is string enterprise)
					{
						webBase = enterprise;
						if (!webBase.StartsWith("http://", StringComparison.OrdinalIgnoreCase) && !webBase.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
							webBase = "https://" + webBase.TrimEnd('/');
						apiBase = webBase + "/api/v3";
					}
					else
					{
						webBase = "https://github.com";
						apiBase = "https://api.github.com";
					}

					const int cacheVersion = 2;

					using var sha1 = SHA1.Create();
					cacheDirectory = cacheDirectory is not null ? Path.GetFullPath(cacheDirectory) : Path.Combine(Path.GetTempPath(), "GitHubDigestBuilderCache");
					cacheDirectory = Path.Combine(cacheDirectory, BitConverter.ToString(sha1.ComputeHash(Encoding.UTF8.GetBytes($"{cacheVersion} {apiBase} {authToken}"))).Replace("-", "")[..16]);
					Directory.CreateDirectory(cacheDirectory);

					// don't process the same event twice
					var handledEventIds = new HashSet<string>();

					// stop when rate limited
					var rateLimitResetUtc = default(DateTime?);

					async Task<PagedDownloadResult> LoadPagesAsync(string url, string[] accepts, int maxPageCount, Func<JsonElement, bool>? isLastPage = null)
					{
						var cacheName = Regex.Replace(Regex.Replace(url, @"\?.*$", ""), @"/+", "_").Trim('_');
						var cacheFile = Path.Combine(cacheDirectory, $"{cacheName}.json");
						var cacheElement = default(JsonElement);
						string? etag = null;

						if (File.Exists(cacheFile))
						{
							await using var cacheReadStream = File.OpenRead(cacheFile);
							cacheElement = (await JsonDocument.ParseAsync(cacheReadStream)).RootElement;

							// if we're asking for the same date, and the cache was generated for a previous day, assume it is still good
							var cacheDateIso = cacheElement.GetProperty("date").GetString();
							var cacheTodayIso = cacheElement.TryGetProperty("today")?.GetString();
							if (cacheDateIso == dateIso && string.CompareOrdinal(cacheDateIso, cacheTodayIso) < 0)
							{
								return new PagedDownloadResult(
									Enum.Parse<DownloadStatus>(cacheElement.GetProperty("status").GetString() ?? throw new InvalidOperationException("Missing status.")),
									cacheElement.GetProperty("items").EnumerateArray().ToList());
							}

							// don't use the cache if we may have stopped early and we're asking for an older report
							if (isLastPage is null || string.CompareOrdinal(dateIso, cacheDateIso) >= 0)
								etag = cacheElement.GetProperty("etag").GetString();
						}

						if (rateLimitResetUtc is object)
							return new PagedDownloadResult(DownloadStatus.RateLimited);

						var status = DownloadStatus.TooMuchActivity;
						var items = new List<JsonElement>();

						for (var pageNumber = 1; pageNumber <= maxPageCount; pageNumber++)
						{
							var pageParameter = pageNumber == 1 ? "" : $"{(url.Contains('?') ? '&' : '?')}page={pageNumber}";
							var request = new HttpRequestMessage(HttpMethod.Get, $"{apiBase}/{url}{pageParameter}");
							foreach (var accept in accepts)
								request.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse(accept));
							if (pageNumber == 1 && etag is not null)
								request.Headers.IfNoneMatch.Add(EntityTagHeaderValue.Parse(etag));

							var response = await httpClient!.SendAsync(request);
							if (isVerbose)
								Console.WriteLine($"{request.RequestUri!.AbsoluteUri} [{response.StatusCode}]");

							if (pageNumber == 1 && etag is not null && response.StatusCode == HttpStatusCode.NotModified)
							{
								status = Enum.Parse<DownloadStatus>(cacheElement.GetProperty("status").GetString() ?? throw new InvalidOperationException("Missing status."));
								items.AddRange(cacheElement.GetProperty("items").EnumerateArray());
								break;
							}

							if (response.StatusCode == HttpStatusCode.NotFound)
								return new PagedDownloadResult(DownloadStatus.NotFound);

							if (response.StatusCode == HttpStatusCode.Forbidden &&
								response.Headers.TryGetValues("X-RateLimit-Remaining", out var rateLimitRemainingValues) &&
								rateLimitRemainingValues.FirstOrDefault() == "0" &&
								response.Headers.TryGetValues("X-RateLimit-Reset", out var rateLimitResetValues) &&
								int.TryParse(rateLimitResetValues.FirstOrDefault() ?? "", out var resetEpochSeconds))
							{
								rateLimitResetUtc = DateTime.UnixEpoch.AddSeconds(resetEpochSeconds);

								var message = "GitHub API rate limit exceeded. " +
									$"Try again at {new DateTimeOffset(rateLimitResetUtc.Value).ToOffset(timeZoneOffset).ToString("f", culture)}.";
								if (authToken is null)
									message += " Specify a GitHub personal access token for a much higher rate limit.";
								AddWarning(message);

								return new PagedDownloadResult(DownloadStatus.RateLimited);
							}

							if (response.StatusCode == HttpStatusCode.Unauthorized)
								throw new ApplicationException("GitHub API returned 401 Unauthorized. Ensure that your auth token is set to a valid personal access token.");

							if (response.StatusCode != HttpStatusCode.OK)
								throw new InvalidOperationException($"Unexpected status code: {response.StatusCode}");

							if (pageNumber == 1)
								etag = (response.Headers.ETag ?? throw new InvalidOperationException("Missing ETag.")).Tag;

							await using var pageStream = await response.Content.ReadAsStreamAsync();
							var pageDocument = await JsonDocument.ParseAsync(pageStream);

							items.AddRange(pageDocument.RootElement.EnumerateArray());

							if (pageDocument.RootElement.GetArrayLength() == 0 || isLastPage?.Invoke(pageDocument.RootElement) == true)
							{
								status = DownloadStatus.Success;
								break;
							}
						}

						await using var cacheWriteStream = File.Open(cacheFile, FileMode.Create, FileAccess.Write);
						await JsonSerializer.SerializeAsync(cacheWriteStream, new
						{
							status = status.ToString(),
							etag,
							date = dateIso,
							today = todayIso,
							items,
						}, new JsonSerializerOptions { WriteIndented = true });

						return new PagedDownloadResult(status, items);
					}

					void AddWarning(string text)
					{
						if (!report!.Warnings.Contains(text))
						{
							report.Warnings.Add(text);
							Console.Error.WriteLine(text);
						}
					}

					var sourceRepoNames = new List<string>();
					var sourceRepoIndices = new Dictionary<string, int>();

					void AddRepoForSource(string repoName, int sourceIndex)
					{
						if (!sourceRepoIndices.ContainsKey(repoName))
						{
							sourceRepoNames.Add(repoName);
							sourceRepoIndices.Add(repoName, sourceIndex);
						}
					}

					async Task AddReposForSource(string sourceKind, string sourceName, int sourceIndex, string? topic)
					{
						var orgRepoNames = new HashSet<string>();
						var result = await LoadPagesAsync($"{sourceKind}/{sourceName}/repos?sort=updated&per_page=100",
							accepts: new[] { "application/vnd.github.mercy-preview+json" }, maxPageCount: 100);

						foreach (var repoElement in result.Elements)
						{
							if (!repoElement.GetProperty("private").GetBoolean() &&
								!repoElement.GetProperty("archived").GetBoolean() &&
								!repoElement.GetProperty("disabled").GetBoolean() &&
								(topic is null || repoElement.GetProperty("topics").EnumerateArray().Select(x => x.GetString()).Any(x => x == topic)))
							{
								orgRepoNames.Add(repoElement.GetProperty("full_name").GetString() ?? throw new InvalidOperationException("Missing full_name."));
							}
						}

						foreach (var orgRepoName in orgRepoNames)
							AddRepoForSource(orgRepoName, sourceIndex);

						if (result.Status == DownloadStatus.TooMuchActivity)
							AddWarning($"Too many updated repositories found for {sourceName}.");
						else if (result.Status == DownloadStatus.NotFound)
							AddWarning($"Failed to find repositories for {sourceName}.");
					}

					var settingsRepos = github.Repos ?? new List<RepoSettings>();
					var settingsUsers = github.Users ?? new List<UserSettings>();
					if (settingsRepos.Count == 0 && settingsUsers.Count == 0)
						throw new ApplicationException("No repositories or users specified in configuration.");

					foreach (var (settingsRepo, sourceIndex) in settingsRepos.Select((x, i) => (x, i)))
					{
						switch (settingsRepo)
						{
							case { Name: string name, User: null, Org: null, Topic: null }:
								AddRepoForSource(name, sourceIndex);
								break;

							case { Name: null, User: string user, Org: null }: