private static void WriteStatToConsole(StatHatStatistic stat)
{
var type = stat.Value != null ? "V" : "C";
object value = stat.Count ?? stat.Value;
Console.WriteLine($"[STATHAT] [{EpochUtility.ToDateTimeOffset(stat.UnixTimestamp):o}] [{stat.Name} ({type})]: {value}");
}
public DateTime?GetDateTime(string key, bool required = true)
{
long?ts = GetValue <long?>(key, required);
if (ts == null)
{
return(null);
}
return(EpochUtility.ToDateTime((int)ts));
}
// /////////////////////////////////////////////////////////////
// Accounting
// /////////////////////////////////////////////////////////////
private static void Record(string statName, long?count, double?value, DateTimeOffset timestamp)
{
if (!StatHatPrefix.IsNullOrWhiteSpace())
{
statName = StatHatPrefix + statName;
}
#if DEBUG
Debug.Assert(!statName.IsNullOrWhiteSpace(), $"Invalid StatHat name: '{statName}'");
Debug.Assert(statName.Length <= 255, $"Invalid StatHat name (too long): '{statName}'");
Debug.Assert(count.HasValue != value.HasValue, $"Statistics must have a count or a value but not both. ({statName})");
#endif
if (StatHatApiKey.IsNullOrWhiteSpace())
{
return;
}
if (statName.IsNullOrWhiteSpace())
{
return;
}
var stat = new StatHatStatistic()
{
Name = statName,
Count = count,
Value = value,
UnixTimestamp = (int)EpochUtility.ToEpoch(timestamp)
};
Sink.Record(stat);
#if DEBUG
WriteStatToConsole(stat);
#endif
}
private async Task <GitHubResponse <T> > MakeRequest <T>(IGitHubClient client, GitHubRequest request, CancellationToken cancellationToken, GitHubRedirect redirect)
{
var uri = new Uri(client.ApiRoot, request.Uri);
var httpRequest = new HttpRequestMessage(request.Method, uri)
{
Content = request.CreateBodyContent(),
};
// Accept header
if (!request.AcceptHeaderOverride.IsNullOrWhiteSpace())
{
httpRequest.Headers.Accept.Clear();
httpRequest.Headers.Accept.ParseAdd(request.AcceptHeaderOverride);
}
else if (typeof(T) == typeof(byte[]))
{
httpRequest.Headers.Accept.Clear();
httpRequest.Headers.Accept.ParseAdd("application/vnd.github.v3.raw");
}
httpRequest.Headers.Authorization = new AuthenticationHeaderValue("bearer", client.AccessToken);
var cache = request.CacheOptions;
if (cache?.UserId == client.UserId)
{
if (request.Method != HttpMethod.Get)
{
throw new InvalidOperationException("Cache options are only valid on GET requests.");
}
httpRequest.Headers.IfModifiedSince = cache.LastModified;
if (!cache.ETag.IsNullOrWhiteSpace())
{
httpRequest.Headers.IfNoneMatch.Add(new EntityTagHeaderValue(cache.ETag));
}
}
// User agent
httpRequest.Headers.UserAgent.Clear();
httpRequest.Headers.UserAgent.Add(client.UserAgent);
// For logging
LoggingMessageProcessingHandler.SetLogDetails(
httpRequest,
client.UserInfo,
$"{client.UserInfo}/{DateTime.UtcNow:o}_{client.NextRequestId()}{httpRequest.RequestUri.PathAndQuery.Replace('/', '_')}.txt",
request.CreationDate
);
HttpResponseMessage response;
var sw = Stopwatch.StartNew();
try {
response = await _HttpClient.SendAsync(httpRequest, cancellationToken);
} catch (TaskCanceledException exception) {
sw.Stop();
exception.Report($"Request aborted for /{request.Uri} after {sw.ElapsedMilliseconds} msec [{LoggingMessageProcessingHandler.ExtractString(httpRequest, LoggingMessageProcessingHandler.LogBlobNameKey)}]");
throw;
}
// Handle redirects
switch (response.StatusCode)
{
case HttpStatusCode.MovedPermanently:
case HttpStatusCode.RedirectKeepVerb:
request = request.CloneWithNewUri(response.Headers.Location);
return(await MakeRequest <T>(client, request, cancellationToken, new GitHubRedirect(response.StatusCode, uri, request.Uri, redirect)));
case HttpStatusCode.Redirect:
case HttpStatusCode.RedirectMethod:
request = request.CloneWithNewUri(response.Headers.Location);
request.Method = HttpMethod.Get;
return(await MakeRequest <T>(client, request, cancellationToken, new GitHubRedirect(response.StatusCode, uri, request.Uri, redirect)));
default:
break;
}
var result = new GitHubResponse <T>(request)
{
Date = response.Headers.Date.Value,
Redirect = redirect,
Status = response.StatusCode,
};
// Cache Headers
result.CacheData = new GitHubCacheDetails()
{
UserId = client.UserId,
Path = request.Path,
AccessToken = client.AccessToken,
ETag = response.Headers.ETag?.Tag,
LastModified = response.Content?.Headers?.LastModified,
PollInterval = response.ParseHeader("X-Poll-Interval", x => (x == null) ? TimeSpan.Zero : TimeSpan.FromSeconds(int.Parse(x))),
};
// Expires and Caching Max-Age
var expires = response.Content?.Headers?.Expires;
var maxAgeSpan = response.Headers.CacheControl?.SharedMaxAge ?? response.Headers.CacheControl?.MaxAge;
if (maxAgeSpan != null)
{
var maxAgeExpires = DateTimeOffset.UtcNow.Add(maxAgeSpan.Value);
if (expires == null || maxAgeExpires < expires)
{
expires = maxAgeExpires;
}
}
result.CacheData.Expires = expires;
// Experiment:
var pollExpires = result.Date.Add(result.CacheData.PollInterval);
if (result.CacheData.Expires > pollExpires)
{
result.CacheData.Expires = pollExpires;
}
// Rate Limits
// These aren't always sent. Check for presence and fail gracefully.
if (response.Headers.Contains("X-RateLimit-Limit"))
{
result.RateLimit = new GitHubRateLimit(
client.AccessToken,
response.ParseHeader("X-RateLimit-Limit", x => int.Parse(x)),
response.ParseHeader("X-RateLimit-Remaining", x => int.Parse(x)),
response.ParseHeader("X-RateLimit-Reset", x => EpochUtility.ToDateTimeOffset(int.Parse(x))));
}
// Abuse
if (response.Headers.RetryAfter != null)
{
var after = response.Headers.RetryAfter;
if (after.Date != null)
{
result.RetryAfter = after.Date;
}
else if (after.Delta != null)
{
result.RetryAfter = DateTimeOffset.UtcNow.Add(after.Delta.Value);
}
}
// Scopes
var scopes = response.ParseHeader <IEnumerable <string> >("X-OAuth-Scopes", x => x?.Split(new[] { ' ', ',' }, StringSplitOptions.RemoveEmptyEntries));
if (scopes != null)
{
result.Scopes.UnionWith(scopes);
}
// Pagination
// Screw the RFC, minimally match what GitHub actually sends.
result.Pagination = response.ParseHeader("Link", x => (x == null) ? null : GitHubPagination.FromLinkHeader(x));
if (result.Succeeded)
{
if (response.StatusCode == HttpStatusCode.NoContent && typeof(T) == typeof(bool))
{
// Gross special case hack for Assignable :/
result.Result = (T)(object)true;
}
else if (response.Content != null && typeof(T) == typeof(byte[]))
{
// Raw byte result
result.Result = (T)(object)(await response.Content.ReadAsByteArrayAsync());
}
else if (response.Content != null && request is GitHubGraphQLRequest)
{
// GraphQL
var resp = await response.Content.ReadAsAsync <GraphQLResponse <T> >(GraphQLSerialization.MediaTypeFormatters);
result.Result = resp.Data;
// Possible to get errors and a (partially) useful response.
if (resp.Errors?.Any() == true)
{
result.Error = new GitHubError()
{
// This is a gross hack.
Message = resp.Errors.SerializeObject(),
};
}
}
else if (response.Content != null)
{
// JSON formatted result
result.Result = await response.Content.ReadAsAsync <T>(GitHubSerialization.MediaTypeFormatters);
// At this point each response represents a single page.
result.Pages = 1;
}
}
else if (response.Content != null)
{
var mediaType = response.Content.Headers.ContentType.MediaType;
if (mediaType.Contains("github") || mediaType.Contains("json"))
{
result.Error = await response.Content.ReadAsAsync <GitHubError>(GitHubSerialization.MediaTypeFormatters);
result.Error.Status = response.StatusCode;
}
else
{
// So far, these have all been nginx errors, mostly unicorns and 502s
// They're already logged by the LoggingMessageProcessingHandler.
var body = await response.Content.ReadAsStringAsync();
Log.Info($"Invalid GitHub Response for [{request.Uri}]:\n\n{body}");
}
}
return(result);
}
protected async override Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var blobName = ExtractString(request, LogBlobNameKey);
var userInfo = ExtractString(request, UserInfoKey);
var timer = new Stopwatch();
HttpResponseMessage response = null;
Exception logException = null;
try {
// Load the request into the buffer so we can copy it later if the request failed.
request.Content?.LoadIntoBufferAsync();
timer.Restart();
response = await base.SendAsync(request, cancellationToken);
timer.Stop();
} catch (Exception e) {
logException = e;
throw;
} finally {
var contentLength = response?.Content?.Headers?.ContentLength ?? 0;
string logBlob = null;
var statusLine = response == null ? "FAILED" : $"{(int)response.StatusCode} {response.ReasonPhrase}";
var skipLogging = logException != null && logException is TaskCanceledException;
if (!skipLogging && IsFailure(response) && _blobClient != null && !blobName.IsNullOrWhiteSpace())
{
logBlob = blobName;
if (!_initialized)
{
await Initialize();
}
using (var ms = new MemoryStream())
using (var sw = new StreamWriter(ms, Encoding.UTF8)) {
await WriteRequest(ms, sw, request);
if (logException != null)
{
sw.WriteLine("\n\nError reading response:\n\n");
sw.WriteLine(logException.ToString());
}
else
{
await WriteResponse(ms, sw, response);
}
_appendBlobs.OnNext(new AppendBlobEntry()
{
BlobName = blobName,
Content = Encoding.UTF8.GetString(ms.ToArray()),
});
}
}
// Rate limit info
var rateInfo = string.Empty;
if (response?.Headers?.Contains("X-RateLimit-Limit") == true)
{
var rateRemaining = response.ParseHeader("X-RateLimit-Remaining", x => int.Parse(x));
var rateReset = response.ParseHeader("X-RateLimit-Reset", x => EpochUtility.ToDateTimeOffset(int.Parse(x)));
rateInfo = $" [{rateRemaining}, {rateReset:o}]";
}
Log.Info($"[{userInfo}] {request.Method} {request.RequestUri.PathAndQuery} HTTP/{request.Version} - {statusLine} - {timer.ElapsedMilliseconds}ms - {ExtractElapsedTime(request)} - {logBlob} - {contentLength} bytes.{rateInfo}");
}
return(response);
}
