/// <inheritdoc />
protected override void ReceiveWithSemaphoreHeld(IEnumerable <T> items)
{
GaxPreconditions.CheckNotNull(items, nameof(items));
_items.AddRange(items);
if (_clock.GetCurrentDateTimeUtc() >= _nextFlush)
{
FlushWithSemaphoreHeld();
}
}
public override Task WriteAsync(StreamingPullRequest message)
{
lock (_lock)
{
_extends.AddRange(message.ModifyDeadlineAckIds.Select(id => new TimedId(_clock.GetCurrentDateTimeUtc(), id)));
_acks.AddRange(message.AckIds.Select(id => new TimedId(_clock.GetCurrentDateTimeUtc(), id)));
return(Task.FromResult(0));
}
}
public override async Task AcknowledgeAsync(SubscriptionName subscription, IEnumerable <string> ackIds, CancellationToken cancellationToken)
{
if (_writeAsyncPreDelay != TimeSpan.Zero)
{
await _taskHelper.ConfigureAwait(_scheduler.Delay(_writeAsyncPreDelay, CancellationToken.None));
}
lock (_lock)
{
_acks.AddRange(ackIds.Select(id => new TimedId(_clock.GetCurrentDateTimeUtc(), id)));
}
}
public override async Task WriteAsync(StreamingPullRequest message)
{
await _taskHelper.ConfigureAwait(_scheduler.Delay(_writeAsyncPreDelay, CancellationToken.None));
var now = _clock.GetCurrentDateTimeUtc();
lock (_lock)
{
_extends.AddRange(message.ModifyDeadlineAckIds.Select(id => new TimedId(now, id)).ToList());
_acks.AddRange(message.AckIds.Select(id => new TimedId(now, id)).ToList());
}
}
// TODO: This is a modified/simplified version of ApiCallRetryExtensions.WithRetry from Google.Api.Gax.Grpc. Can we combine them somehow?
internal static async Task RetryOperationUntilCompleted(
Func <Task <bool> > fn,
IClock clock,
IScheduler scheduler,
CallSettings callSettings,
RetrySettings retrySettings)
{
DateTime?overallDeadline = retrySettings.TotalExpiration.CalculateDeadline(clock);
TimeSpan retryDelay = retrySettings.RetryBackoff.Delay;
TimeSpan callTimeout = retrySettings.TimeoutBackoff.Delay;
while (true)
{
DateTime attemptDeadline = clock.GetCurrentDateTimeUtc() + callTimeout;
// Note: this handles a null total deadline due to "<" returning false if overallDeadline is null.
DateTime combinedDeadline = overallDeadline < attemptDeadline ? overallDeadline.Value : attemptDeadline;
CallSettings attemptCallSettings = callSettings.WithCallTiming(CallTiming.FromDeadline(combinedDeadline));
TimeSpan actualDelay = retrySettings.DelayJitter.GetDelay(retryDelay);
DateTime expectedRetryTime;
try
{
bool isResponseOk = await fn().ConfigureAwait(false);
if (isResponseOk)
{
return;
}
expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > overallDeadline)
{
// TODO: Can we get this string from somewhere?
throw new RpcException(new Status(StatusCode.DeadlineExceeded, "Deadline Exceeded"));
}
}
catch (RpcException e) when(retrySettings.RetryFilter(e))
{
expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > overallDeadline)
{
throw;
}
}
await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);
retryDelay = retrySettings.RetryBackoff.NextDelay(retryDelay);
callTimeout = retrySettings.TimeoutBackoff.NextDelay(callTimeout);
}
}
/// <summary>
/// Asynchronously repeatedly calls the specified polling action, delaying between calls,
/// until a given condition is met in the response.
/// </summary>
/// <typeparam name="TResponse">The response type.</typeparam>
/// <param name="pollAction">The poll action, typically performing an RPC. The value passed to the
/// action is the overall deadline, so that the RPC settings can be adjusted accordingly. A null
/// value indicates no deadline.</param>
/// <param name="completionPredicate">The test for whether to return the response (<c>true</c>) or continue
/// polling (<c>false</c>). Must not be null.</param>
/// <param name="clock">The clock to use for determining deadlines. Must not be null.</param>
/// <param name="scheduler">The scheduler to use for delaying between calls. Must not be null.</param>
/// <param name="pollSettings">The poll settings, controlling timeouts, call settings and delays.</param>
/// <param name="cancellationToken">The cancellation token used to cancel delays, if any.</param>
/// <returns>A task representing the asynchronous operation. The result of the task will be the completed response.</returns>
public static async Task <TResponse> PollRepeatedlyAsync <TResponse>(
Func <DateTime?, Task <TResponse> > pollAction,
Predicate <TResponse> completionPredicate,
IClock clock,
IScheduler scheduler,
PollSettings pollSettings,
CancellationToken cancellationToken)
{
GaxPreconditions.CheckNotNull(pollAction, nameof(pollAction));
GaxPreconditions.CheckNotNull(completionPredicate, nameof(completionPredicate));
GaxPreconditions.CheckNotNull(clock, nameof(clock));
GaxPreconditions.CheckNotNull(scheduler, nameof(scheduler));
GaxPreconditions.CheckNotNull(pollSettings, nameof(pollSettings));
var deadline = pollSettings.Expiration.CalculateDeadline(clock);
var currentDelay = pollSettings.Delay;
while (true)
{
var latest = await pollAction(deadline).ConfigureAwait(false);
if (completionPredicate(latest))
{
return(latest);
}
if (clock.GetCurrentDateTimeUtc() + pollSettings.Delay >= deadline)
{
// TODO: Could return null instead. Unclear what's better here.
throw new TimeoutException("Operation did not complete within the specified expiry time");
}
await scheduler.Delay(currentDelay, cancellationToken).ConfigureAwait(false);
currentDelay = pollSettings.NextDelay(currentDelay);
}
}
public override Task WriteAsync(StreamingPullRequest message)
{
if (message.ModifyDeadlineAckIds.Count != 0 || message.AckIds.Count != 0)
{
throw new InvalidOperationException("WriteAsync must not modify deadlines or send acks/nacks.");
}
if (Equals(message, new StreamingPullRequest()))
{
// An empty message is a ping
lock (_lock)
{
_streamPings.Add(_clock.GetCurrentDateTimeUtc());
}
}
return(Task.FromResult(0));
}
/// <inheritdoc />
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
{
GaxPreconditions.CheckNotNull(formatter, nameof(formatter));
if (!IsEnabled(logLevel))
{
return;
}
string message = formatter(state, exception);
if (string.IsNullOrEmpty(message))
{
return;
}
LogEntry entry = new LogEntry
{
Resource = _loggerOptions.MonitoredResource,
LogName = _logName,
Severity = logLevel.ToLogSeverity(),
Timestamp = Timestamp.FromDateTime(_clock.GetCurrentDateTimeUtc()),
TextPayload = string.Concat(GoogleLoggerScope.Current, message),
Labels = { _loggerOptions.Labels },
Trace = GetTraceName() ?? "",
};
_consumer.Receive(new[] { entry });
}
internal async Task <TResult> ExecuteWithRetryAsync <TResult>(Func <Task <TResult> > fn, CancellationToken cancellationToken)
{
// Note: the way that we sometimes use the recommended retry delay, and base
// further delays on that, means we can't use RetryAttempt easily here.
DateTime?overallDeadline = _options.CalculateDeadline(_clock);
TimeSpan retryDelay = _options.InitialDelay;
while (true)
{
try
{
var result = await fn().ConfigureAwait(false);
return(result);
}
catch (SpannerException e) when(e.IsRetryable && !e.SessionExpired)
{
// If there's a recommended retry delay specified on the exception
// we should respect it.
retryDelay = e.RecommendedRetryDelay ?? retryDelay;
TimeSpan actualDelay = _options.Jitter(retryDelay);
DateTime expectedRetryTime = _clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > overallDeadline)
{
throw;
}
await _scheduler.Delay(actualDelay, cancellationToken).ConfigureAwait(false);
retryDelay = _options.NextDelay(retryDelay);
}
}
}
private TimedBufferingConsumer(IConsumer <T> consumer, TimeSpan waitTime, IClock clock)
{
_consumer = GaxPreconditions.CheckNotNull(consumer, nameof(consumer));
_waitTime = waitTime;
_clock = GaxPreconditions.CheckNotNull(clock, nameof(clock));
_nextFlush = _clock.GetCurrentDateTimeUtc().Add(_waitTime);
}
internal static async Task <TResponse> CallWithRetryAsync <TResponse>(
this Func <Task <TResponse> > someMethod,
CallSettings callSettings,
IClock clock, IScheduler scheduler)
{
RetrySettings retrySettings = callSettings.Timing?.Retry;
if (retrySettings == null)
{
return(await someMethod().ConfigureAwait(false));
}
DateTime?overallDeadline = retrySettings.TotalExpiration.CalculateDeadline(clock);
TimeSpan retryDelay = retrySettings.RetryBackoff.Delay;
while (true)
{
try
{
return(await someMethod().ConfigureAwait(false));
}
catch (RpcException e) when(retrySettings.RetryFilter(e))
{
TimeSpan actualDelay = retrySettings.DelayJitter.GetDelay(retryDelay);
DateTime expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > overallDeadline)
{
throw;
}
await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);
retryDelay = retrySettings.RetryBackoff.NextDelay(retryDelay);
}
}
}
/// <inheritdoc />
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
{
try
{
GaxPreconditions.CheckNotNull(formatter, nameof(formatter));
if (!IsEnabled(logLevel))
{
return;
}
string message = formatter(state, exception);
if (string.IsNullOrEmpty(message))
{
return;
}
LogEntry entry = new LogEntry
{
Resource = _loggerOptions.MonitoredResource,
LogName = _fullLogName,
Severity = logLevel.ToLogSeverity(),
Timestamp = Timestamp.FromDateTime(_clock.GetCurrentDateTimeUtc()),
JsonPayload = CreateJsonPayload(eventId, state, exception, message),
Labels = { CreateLabels() },
Trace = GetTraceName() ?? "",
};
_consumer.Receive(new[] { entry });
}
catch (Exception) when(_loggerOptions.RetryOptions.ExceptionHandling == ExceptionHandling.Ignore)
{
}
}
/// <inheritdoc />
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
{
GaxPreconditions.CheckNotNull(formatter, nameof(formatter));
if (!IsEnabled(logLevel))
{
return;
}
string message = formatter(state, exception);
if (string.IsNullOrEmpty(message))
{
return;
}
LogEntry entry = new LogEntry
{
Resource = _globalResource,
LogName = _logName,
Severity = logLevel.ToLogSeverity(),
Timestamp = Timestamp.FromDateTime(_clock.GetCurrentDateTimeUtc()),
TextPayload = message,
};
_consumer.Receive(new[] { entry });
}
// By design, the code is mostly duplicated between the async and sync calls.
// Async retry
internal static Func <TRequest, CallSettings, Task <TResponse> > WithRetry <TRequest, TResponse>(
this Func <TRequest, CallSettings, Task <TResponse> > fn,
IClock clock, IScheduler scheduler, Func <TResponse, Task> postResponse) =>
async(request, callSettings) =>
{
RetrySettings retrySettings = callSettings.Timing?.Retry;
if (retrySettings == null)
{
return(await fn(request, callSettings).ConfigureAwait(false));
}
DateTime?overallDeadline = retrySettings.TotalExpiration.CalculateDeadline(clock);
TimeSpan retryDelay = retrySettings.RetryBackoff.Delay;
TimeSpan callTimeout = retrySettings.TimeoutBackoff.Delay;
while (true)
{
DateTime attemptDeadline = clock.GetCurrentDateTimeUtc() + callTimeout;
// Note: this handles a null total deadline due to "<" returning false if overallDeadline is null.
DateTime combinedDeadline = overallDeadline < attemptDeadline ? overallDeadline.Value : attemptDeadline;
CallSettings attemptCallSettings = callSettings.WithCallTiming(CallTiming.FromDeadline(combinedDeadline));
try
{
var response = await fn(request, attemptCallSettings).ConfigureAwait(false);
if (postResponse != null)
{
await postResponse(response).ConfigureAwait(false);
}
return(response);
}
catch (RpcException e) when(retrySettings.RetryFilter(e))
{
TimeSpan actualDelay = retrySettings.DelayJitter.GetDelay(retryDelay);
DateTime expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > overallDeadline)
{
throw;
}
await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);
retryDelay = retrySettings.RetryBackoff.NextDelay(retryDelay);
callTimeout = retrySettings.TimeoutBackoff.NextDelay(callTimeout);
}
}
};
internal PostPolicySigningState(PostPolicy policy, Options options, IBlobSigner blobSigner, IClock clock)
{
string uri = options.UrlStyle switch
{
UrlStyle.PathStyle => policy.Bucket == null ? StorageHost : $"{StorageHost}/{policy.Bucket}",
UrlStyle.VirtualHostedStyle => policy.Bucket == null ?
throw new ArgumentNullException(nameof(PostPolicy.Bucket), $"When using {UrlStyle.VirtualHostedStyle} a bucket condition must be set in the policy.") :
$"{policy.Bucket}.{StorageHost}",
UrlStyle.BucketBoundHostname => options.BucketBoundHostname,
_ => throw new ArgumentOutOfRangeException(nameof(options.UrlStyle))
};
uri = $"{options.Scheme}://{uri}/";
options = options.ToExpiration(clock);
var now = clock.GetCurrentDateTimeUtc();
int expirySeconds = (int)(options.Expiration.Value - now).TotalSeconds;
if (expirySeconds <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options.Expiration), "Expiration must be at least 1 second.");
}
if (expirySeconds > MaxExpirySecondsInclusive)
{
throw new ArgumentOutOfRangeException(nameof(options.Expiration), "Expiration must not be greater than 7 days.");
}
var datestamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
string credentialScope = $"{datestamp}/{DefaultRegion}/{ScopeSuffix}";
policy.SetField(PolicyCreationDateTime.Element, new DateTimeOffset(now));
policy.SetField(PolicyAlgorithm.Element, Algorithm);
policy.SetField(PolicyCredential.Element, $"{blobSigner.Id}/{credentialScope}");
_expiration = options.Expiration.Value;
_policy = policy;
_url = new Uri(uri);
StringBuilder sb = new StringBuilder();
StringWriter sw = new StringWriter(sb);
using (JsonWriter writer = new JsonTextWriter(sw))
{
writer.StringEscapeHandling = StringEscapeHandling.EscapeNonAscii;
writer.WriteStartObject();
policy.WriteTo(writer);
writer.WritePropertyName("expiration");
writer.WriteValue(_expiration.UtcDateTime.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'", CultureInfo.InvariantCulture));
writer.WriteEndObject();
}
var decodedPolicy = sb.ToString();
_encodedPolicy = Convert.ToBase64String(Encoding.UTF8.GetBytes(decodedPolicy));
_blobToSign = Encoding.UTF8.GetBytes(_encodedPolicy);
}
private LogEntry MakeLostEntry(DateTimeRange lostRange)
{
var lostEntry = _logsLostWarningTemplate.Clone();
lostEntry.Timestamp = _clock.GetCurrentDateTimeUtc().ToTimestamp();
lostEntry.TextPayload = string.Format(lostEntry.TextPayload,
lostRange.From.ToString(GoogleStackdriverAppender.s_lostDateTimeFmt),
lostRange.To.ToString(GoogleStackdriverAppender.s_lostDateTimeFmt));
return(lostEntry);
}
internal async Task RetryAsync(SpannerException abortedException, CancellationToken cancellationToken, int timeoutSeconds = MAX_TIMEOUT_SECONDS)
{
if (!EnableInternalRetries)
{
throw abortedException;
}
DateTime?overallDeadline = _options.CalculateDeadline(_clock);
TimeSpan retryDelay = _options.InitialDelay;
// If there's a recommended retry delay specified on the exception
// we should respect it.
retryDelay = abortedException.RecommendedRetryDelay ?? _options.Jitter(retryDelay);
while (true)
{
if (RetryCount >= MaxInternalRetryCount)
{
throw new SpannerException(ErrorCode.Aborted, "Transaction was aborted because it aborted and retried too many times");
}
DateTime expectedRetryTime = _clock.GetCurrentDateTimeUtc() + retryDelay;
if (expectedRetryTime > overallDeadline)
{
throw new SpannerException(ErrorCode.Aborted, "Transaction was aborted because it timed out while retrying");
}
await _scheduler.Delay(retryDelay, cancellationToken).ConfigureAwait(false);
// TODO: Preferably the Spanner client library should have some 'reset' option on an existing
// transaction instead of having to begin a new transaction. This will potentially use a transaction
// on a different session, while the recommended behavior for a retry is to use the same session as
// the original attempt.
SpannerTransaction.Dispose();
SpannerTransaction = await Connection.SpannerConnection.BeginTransactionAsync(cancellationToken);
RetryCount++;
try
{
foreach (IRetriableStatement statement in _retriableStatements)
{
await statement.RetryAsync(this, cancellationToken, timeoutSeconds);
}
break;
}
catch (SpannerAbortedDueToConcurrentModificationException)
{
// Retry failed because of a concurrent modification.
throw;
}
catch (SpannerException e) when(e.ErrorCode == ErrorCode.Aborted)
{
// Ignore and retry.
retryDelay = e.RecommendedRetryDelay ?? _options.Jitter(_options.NextDelay(retryDelay));
}
}
}
/// <inheritdoc />
public void Receive(IEnumerable <T> items)
{
GaxPreconditions.CheckNotNull(items, nameof(items));
lock (_mutex)
{
_items.AddRange(items);
if (_clock.GetCurrentDateTimeUtc() >= _nextFlush)
{
Flush();
}
}
}
/// <summary>
/// Updates the state on the basis of the given exception, delaying for as long as is necessary
/// between retries.
/// </summary>
internal async Task WaitAsync(RpcException exception, CancellationToken cancellationToken)
{
TimeSpan?delayFromException = GetRetryDelay(exception);
TimeSpan delay = _retrySettings.BackoffJitter.GetDelay(delayFromException ?? _retrySettingsBackoffs.Current);
if (_clock.GetCurrentDateTimeUtc() + delay >= _currentDeadline)
{
throw new RpcException(new Grpc.Core.Status(StatusCode.DeadlineExceeded, "Streaming call exceeded timeout"));
}
await _scheduler.Delay(delay, cancellationToken).ConfigureAwait(false);
if (delayFromException == null)
{
_retrySettingsBackoffs.MoveNext();
}
}
// By design, the code is mostly duplicated between the async and sync calls.
// Async retry
internal static Func <TRequest, CallSettings, Task <TResponse> > WithRetry <TRequest, TResponse>(
this Func <TRequest, CallSettings, Task <TResponse> > fn,
IClock clock, IScheduler scheduler, Func <TResponse, Task> postResponse) =>
async(request, callSettings) =>
{
RetrySettings retrySettings = callSettings.Retry;
if (retrySettings == null)
{
return(await fn(request, callSettings).ConfigureAwait(false));
}
DateTime?overallDeadline = callSettings.Expiration.CalculateDeadline(clock);
// Every attempt should use the same deadline, calculated from the start of the call.
if (callSettings.Expiration?.Type == ExpirationType.Timeout)
{
callSettings = callSettings.WithDeadline(overallDeadline.Value);
}
TimeSpan backoffDelay = retrySettings.InitialBackoff;
int attempt = 0;
while (true)
{
try
{
attempt++;
var response = await fn(request, callSettings).ConfigureAwait(false);
if (postResponse != null)
{
await postResponse(response).ConfigureAwait(false);
}
return(response);
}
catch (RpcException e) when(attempt < retrySettings.MaxAttempts && retrySettings.RetryFilter(e))
{
TimeSpan actualDelay = retrySettings.BackoffJitter.GetDelay(backoffDelay);
DateTime expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > overallDeadline)
{
throw;
}
await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);
backoffDelay = retrySettings.NextBackoff(backoffDelay);
}
}
};
private void ActivateLogIdAndResource()
{
string projectId = null;
MonitoredResource resource = null;
if (!DisableResourceTypeDetection)
{
resource = MonitoredResourceBuilder.FromPlatform(_platform);
resource.Labels.TryGetValue("project_id", out projectId);
}
if (projectId == null)
{
// Either platform detection is disabled, or it detected an unknown platform
// So use the manually configured projectId and override the resource
projectId = GaxPreconditions.CheckNotNull(ProjectId, nameof(ProjectId));
if (ResourceType == null)
{
resource = new MonitoredResource {
Type = "global", Labels = { { "project_id", projectId } }
};
}
else
{
resource = new MonitoredResource {
Type = ResourceType, Labels = { _resourceLabels.ToDictionary(x => x.Key, x => x.Value) }
};
}
}
else
{
if (ProjectId != null && projectId != ProjectId)
{
_preLogs.Add(() => BuildLogEntry(new LoggingEvent(new LoggingEventData
{
Level = Level.Warn,
LoggerName = "",
Message = s_mismatchedProjectIdMessage,
TimeStamp = _clock.GetCurrentDateTimeUtc()
})));
}
}
_resource = resource;
_logName = new LogName(projectId, LogId).ToString();
}
/// <summary>
/// Calculate a deadline from an <see cref="Expiration"/> and a <see cref="IClock"/>.
/// </summary>
/// <param name="expiration"><see cref="Expiration"/>, may be null.</param>
/// <param name="clock"><see cref="IClock"/> to use for deadline calculation.</param>
/// <returns>The calculated absolute deadline, or null if no deadline should be used.</returns>
internal static DateTime?CalculateDeadline(this Expiration expiration, IClock clock)
{
GaxPreconditions.CheckNotNull(clock, nameof(clock));
if (expiration == null)
{
return(null);
}
switch (expiration.Type)
{
case ExpirationType.None:
return(null);
case ExpirationType.Timeout:
return(clock.GetCurrentDateTimeUtc() + expiration.Timeout.Value);
case ExpirationType.Deadline:
return(expiration.Deadline.Value);
default:
throw new ArgumentException("Invalid expiration", nameof(expiration));
}
}
/// <summary>
/// Extracts the cache value from a Firestore document snapshot.
/// Checks all the expiration fields so never returns a stale value.
/// </summary>
/// <param name="snapshot">The snapshot to pull the cache value from.</param>
/// <returns>The cache value or null.</returns>
private byte[] ValueFromSnapshot(DocumentSnapshot snapshot)
{
if (!snapshot.Exists)
{
return(null);
}
CacheDoc doc = snapshot.ConvertTo <CacheDoc>();
var now = _clock.GetCurrentDateTimeUtc();
if (doc.AbsoluteExpiration < now)
{
return(null);
}
var slidingExpiration = doc.LastRefresh.GetValueOrDefault()
+ TimeSpan.FromSeconds(doc.SlidingExpirationSeconds.GetValueOrDefault());
if (slidingExpiration < now)
{
return(null);
}
return(doc.Value);
}
/// <inheritdoc />
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
{
try
{
GaxPreconditions.CheckNotNull(formatter, nameof(formatter));
if (!IsEnabled(logLevel))
{
return;
}
string message = formatter(state, exception);
if (string.IsNullOrEmpty(message))
{
return;
}
var jsonStruct = new Struct();
jsonStruct.Fields.Add("message", Value.ForString(message));
jsonStruct.Fields.Add("log_name", Value.ForString(_logName));
if (eventId.Id != 0 || eventId.Name != null)
{
var eventStruct = new Struct();
if (eventId.Id != 0)
{
eventStruct.Fields.Add("id", Value.ForNumber(eventId.Id));
}
if (!string.IsNullOrWhiteSpace(eventId.Name))
{
eventStruct.Fields.Add("name", Value.ForString(eventId.Name));
}
jsonStruct.Fields.Add("event_id", Value.ForStruct(eventStruct));
}
// If we have format params and its more than just the original message add them.
if (state is IEnumerable <KeyValuePair <string, object> > formatParams &&
!(formatParams.Count() == 1 && formatParams.Single().Key.Equals("{OriginalFormat}")))
{
var paramStruct = new Struct();
foreach (var pair in formatParams)
{
// Consider adding formatting support for values that are IFormattable.
paramStruct.Fields[pair.Key] = Value.ForString(pair.Value?.ToString() ?? "");
}
if (paramStruct.Fields.Count > 0)
{
jsonStruct.Fields.Add("format_parameters", Value.ForStruct(paramStruct));
}
}
var currentLogScope = GoogleLoggerScope.Current;
if (currentLogScope != null)
{
jsonStruct.Fields.Add("scope", Value.ForString(currentLogScope.ToString()));
}
// Create a map of format parameters of all the parent scopes,
// starting from the most inner scope to the top-level scope.
var scopeParamsList = new List <Value>();
while (currentLogScope != null)
{
// Determine if the state of the scope are format params
if (currentLogScope.State is FormattedLogValues scopeFormatParams)
{
var scopeParams = new Struct();
foreach (var pair in scopeFormatParams)
{
scopeParams.Fields[pair.Key] = Value.ForString(pair.Value?.ToString() ?? "");
}
scopeParamsList.Add(Value.ForStruct(scopeParams));
}
currentLogScope = currentLogScope.Parent;
}
if (scopeParamsList.Count > 0)
{
jsonStruct.Fields.Add("parent_scopes", Value.ForList(scopeParamsList.ToArray()));
}
Dictionary <string, string> labels;
var labelProviders = GetLabelProviders()?.ToArray();
if (labelProviders?.Length > 0)
{
// Create a copy of the labels from the options and invoke each provider
labels = _loggerOptions.Labels.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
foreach (var provider in labelProviders)
{
provider.Invoke(labels);
}
}
else
{
labels = _loggerOptions.Labels;
}
LogEntry entry = new LogEntry
{
Resource = _loggerOptions.MonitoredResource,
LogName = _fullLogName,
Severity = logLevel.ToLogSeverity(),
Timestamp = Timestamp.FromDateTime(_clock.GetCurrentDateTimeUtc()),
JsonPayload = jsonStruct,
Labels = { labels },
Trace = GetTraceName() ?? "",
};
_consumer.Receive(new[] { entry });
}
catch (Exception) when(_loggerOptions.RetryOptions.ExceptionHandling == ExceptionHandling.Ignore)
{
}
}
/// <summary> /// If this set of options was duration based, this method will return a new set /// of options whose expiration will be calculated based on this instance duration /// and the given clock. /// If this set of options was expiration based this same instance will be returned. /// </summary> /// <param name="clock">The clock to use to calculate the expiration.</param> /// <returns>An expiration based set of options.</returns> internal Options ToExpiration(IClock clock) => Expiration.HasValue ? this : WithExpiration(clock.GetCurrentDateTimeUtc() + Duration.Value);
internal UrlSigningState(RequestTemplate template, Options options, IBlobSigner blobSigner, IClock clock)
{
(_host, _resourcePath) = options.UrlStyle switch
{
UrlStyle.PathStyle => (StorageHost, $"/{template.Bucket}"),
UrlStyle.VirtualHostedStyle => ($"{template.Bucket}.{StorageHost}", string.Empty),
UrlStyle.BucketBoundHostname => (options.BucketBoundHostname, string.Empty),
_ => throw new ArgumentOutOfRangeException(nameof(options.UrlStyle))
};
_scheme = options.Scheme;
options = options.ToExpiration(clock);
var now = clock.GetCurrentDateTimeUtc();
var timestamp = now.ToString("yyyyMMdd'T'HHmmss'Z'", CultureInfo.InvariantCulture);
var datestamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
int expirySeconds = (int)(options.Expiration.Value - now).TotalSeconds;
if (expirySeconds <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options.Expiration), "Expiration must be at least 1 second");
}
if (expirySeconds > MaxExpirySecondsInclusive)
{
throw new ArgumentOutOfRangeException(nameof(options.Expiration), "Expiration must not be greater than 7 days.");
}
string expiryText = expirySeconds.ToString(CultureInfo.InvariantCulture);
string credentialScope = $"{datestamp}/{DefaultRegion}/{ScopeSuffix}";
var headers = new SortedDictionary <string, string>(StringComparer.Ordinal);
headers.AddHeader("host", _host);
headers.AddHeaders(template.RequestHeaders);
headers.AddHeaders(template.ContentHeaders);
var canonicalHeaders = string.Join("", headers.Select(pair => $"{pair.Key}:{pair.Value}\n"));
var signedHeaders = string.Join(";", headers.Keys.Select(k => k.ToLowerInvariant()));
var queryParameters = new SortedSet <string>(StringComparer.Ordinal);
queryParameters.AddQueryParameter("X-Goog-Algorithm", Algorithm);
queryParameters.AddQueryParameter("X-Goog-Credential", $"{blobSigner.Id}/{credentialScope}");
queryParameters.AddQueryParameter("X-Goog-Date", timestamp);
queryParameters.AddQueryParameter("X-Goog-Expires", expirySeconds.ToString(CultureInfo.InvariantCulture));
queryParameters.AddQueryParameter("X-Goog-SignedHeaders", signedHeaders);
var effectiveRequestMethod = template.HttpMethod;
if (effectiveRequestMethod == ResumableHttpMethod)
{
effectiveRequestMethod = HttpMethod.Post;
queryParameters.AddQueryParameter("X-Goog-Resumable", "Start");
}
queryParameters.AddQueryParameters(template.QueryParameters);
_canonicalQueryString = string.Join("&", queryParameters);
if (!string.IsNullOrEmpty(template.ObjectName))
{
// EscapeDataString escapes slashes, which we *don't* want to escape here. The simplest option is to
// split the path into segments by slashes, escape each segment, then join the escaped segments together again.
var segments = template.ObjectName.Split('/');
var escaped = string.Join("/", segments.Select(Uri.EscapeDataString));
_resourcePath = _resourcePath + "/" + escaped;
}
string payloadHash = "UNSIGNED-PAYLOAD";
var payloadHashHeader = headers.Where(
header => header.Key.Equals("X-Goog-Content-SHA256", StringComparison.OrdinalIgnoreCase)).ToList();
if (payloadHashHeader.Count == 1)
{
payloadHash = payloadHashHeader[0].Value;
}
var canonicalRequest = $"{effectiveRequestMethod}\n{_resourcePath}\n{_canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\n{payloadHash}";
string hashHex;
using (var sha256 = SHA256.Create())
{
hashHex = FormatHex(sha256.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest)));
}
_blobToSign = Encoding.UTF8.GetBytes($"{Algorithm}\n{timestamp}\n{credentialScope}\n{hashHex}");
}
internal SigningState(
string bucket,
string objectName,
DateTimeOffset expiration,
HttpMethod requestMethod,
Dictionary <string, IEnumerable <string> > requestHeaders,
Dictionary <string, IEnumerable <string> > contentHeaders,
IBlobSigner blobSigner,
IClock clock)
{
StorageClientImpl.ValidateBucketName(bucket);
var now = clock.GetCurrentDateTimeUtc();
var timestamp = now.ToString("yyyyMMdd'T'HHmmss'Z'", CultureInfo.InvariantCulture);
var datestamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
int expirySeconds = (int)(expiration - now).TotalSeconds;
if (expirySeconds <= 0)
{
throw new ArgumentOutOfRangeException(nameof(expiration), "Expiration must be at least 1 second");
}
if (expirySeconds > MaxExpirySecondsInclusive)
{
throw new ArgumentOutOfRangeException(nameof(expiration), "Expiration must not be greater than 7 days.");
}
string expiryText = expirySeconds.ToString(CultureInfo.InvariantCulture);
string credentialScope = $"{datestamp}/{DefaultRegion}/{ScopeSuffix}";
var headers = new SortedDictionary <string, string>(StringComparer.Ordinal);
headers["host"] = HostHeaderValue;
AddHeaders(headers, requestHeaders);
AddHeaders(headers, contentHeaders);
var canonicalHeaders = string.Join("", headers.Select(pair => $"{pair.Key}:{pair.Value}\n"));
var signedHeaders = string.Join(";", headers.Keys.Select(k => k.ToLowerInvariant()));
var queryParameters = new SortedDictionary <string, string>(StringComparer.Ordinal)
{
{ "X-Goog-Algorithm", Algorithm },
{ "X-Goog-Credential", $"{blobSigner.Id}/{credentialScope}" },
{ "X-Goog-Date", timestamp },
{ "X-Goog-Expires", expirySeconds.ToString(CultureInfo.InvariantCulture) },
{ "X-Goog-SignedHeaders", signedHeaders }
};
if (requestMethod == null)
{
requestMethod = HttpMethod.Get;
}
else if (requestMethod == ResumableHttpMethod)
{
requestMethod = HttpMethod.Post;
queryParameters["X-Goog-Resumable"] = "Start";
}
_canonicalQueryString = string.Join("&", queryParameters.Select(p => $"{p.Key}={Uri.EscapeDataString(p.Value)}"));
_resourcePath = $"/{bucket}";
if (!string.IsNullOrEmpty(objectName))
{
// EscapeDataString escapes slashes, which we *don't* want to escape here. The simplest option is to
// split the path into segments by slashes, escape each segment, then join the escaped segments together again.
var segments = objectName.Split('/');
var escaped = string.Join("/", segments.Select(Uri.EscapeDataString));
_resourcePath = _resourcePath + "/" + escaped;
}
var canonicalRequest = $"{requestMethod}\n{_resourcePath}\n{_canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\nUNSIGNED-PAYLOAD";
string hashHex;
using (var sha256 = SHA256.Create())
{
hashHex = FormatHex(sha256.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest)));
}
_blobToSign = Encoding.UTF8.GetBytes($"{Algorithm}\n{timestamp}\n{credentialScope}\n{hashHex}");
void AddHeaders(SortedDictionary <string, string> canonicalized, IDictionary <string, IEnumerable <string> > headersToAdd)
{
if (headersToAdd == null)
{
return;
}
foreach (var pair in headersToAdd)
{
if (pair.Value == null)
{
continue;
}
var headerName = pair.Key.ToLowerInvariant();
// Note: the comma-space separating here is because this is what HttpClient does.
// Google Cloud Storage itself will just use commas if it receives multiple values for the same header name,
// but HttpClient coalesces the values itself. This approach means that if the same request is made from .NET
// with the signed URL, it will succeed - but it does mean that the signed URL won't be valid when used from
// another platform that sends actual multiple values.
var value = string.Join(", ", pair.Value.Select(PrepareHeaderValue)).Trim();
if (canonicalized.TryGetValue(headerName, out var existingValue))
{
value = $"{existingValue}, {value}";
}
canonicalized[headerName] = value;
}
}
}
/// <summary>
/// Calculate a deadline from an <see cref="Expiration"/> and a <see cref="IClock"/>.
/// </summary>
/// <param name="expiration"><see cref="Expiration"/>, may be null.</param>
/// <param name="clock"><see cref="IClock"/> to use for deadline calculation.</param>
/// <returns>The calculated absolute deadline, or null if no deadline should be used.</returns>
internal static DateTime? CalculateDeadline(this Expiration expiration, IClock clock)
{
GaxPreconditions.CheckNotNull(clock, nameof(clock));
if (expiration == null)
{
return null;
}
switch (expiration.Type)
{
case ExpirationType.None:
return null;
case ExpirationType.Timeout:
return clock.GetCurrentDateTimeUtc() + expiration.Timeout.Value;
case ExpirationType.Deadline:
return expiration.Deadline.Value;
default:
throw new ArgumentException("Invalid expiration", nameof(expiration));
}
}
internal SigningState(
string bucket,
string objectName,
DateTimeOffset expiration,
HttpMethod requestMethod,
Dictionary <string, IEnumerable <string> > requestHeaders,
Dictionary <string, IEnumerable <string> > contentHeaders,
IBlobSigner blobSigner,
IClock clock)
{
StorageClientImpl.ValidateBucketName(bucket);
bool isResumableUpload = false;
if (requestMethod == null)
{
requestMethod = HttpMethod.Get;
}
else if (requestMethod == ResumableHttpMethod)
{
isResumableUpload = true;
requestMethod = HttpMethod.Post;
}
var now = clock.GetCurrentDateTimeUtc();
var timestamp = now.ToString("yyyyMMdd'T'HHmmss'Z'", CultureInfo.InvariantCulture);
var datestamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
// TODO: Validate again maximum expirary duration
int expirySeconds = (int)(expiration - now).TotalSeconds;
string expiryText = expirySeconds.ToString(CultureInfo.InvariantCulture);
string clientEmail = blobSigner.Id;
string credentialScope = $"{datestamp}/auto/gcs/goog4_request";
string credential = WebUtility.UrlEncode($"{blobSigner.Id}/{credentialScope}");
// FIXME: Use requestHeaders and contentHeaders
var headers = new SortedDictionary <string, string>();
headers["host"] = "storage.googleapis.com";
var canonicalHeaderBuilder = new StringBuilder();
foreach (var pair in headers)
{
canonicalHeaderBuilder.Append($"{pair.Key}:{pair.Value}\n");
}
var canonicalHeaders = canonicalHeaderBuilder.ToString().ToLowerInvariant();
var signedHeaders = string.Join(";", headers.Keys.Select(k => k.ToLowerInvariant()));
queryParameters = new List <string>
{
"X-Goog-Algorithm=GOOG4-RSA-SHA256",
$"X-Goog-Credential={credential}",
$"X-Goog-Date={timestamp}",
$"X-Goog-Expires={expirySeconds}",
$"X-Goog-SignedHeaders={signedHeaders}"
};
if (isResumableUpload)
{
queryParameters.Insert(4, "X-Goog-Resumable=Start");
}
var canonicalQueryString = string.Join("&", queryParameters);
resourcePath = $"/{bucket}";
if (objectName != null)
{
resourcePath += $"/{Uri.EscapeDataString(objectName)}";
}
var canonicalRequest = $"{requestMethod}\n{resourcePath}\n{canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\nUNSIGNED-PAYLOAD";
string hashHex;
using (var sha256 = SHA256.Create())
{
hashHex = FormatHex(sha256.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest)));
}
blobToSign = Encoding.UTF8.GetBytes($"GOOG4-RSA-SHA256\n{timestamp}\n{credentialScope}\n{hashHex}");
}
internal SigningState(
string bucket,
string objectName,
DateTimeOffset expiration,
HttpMethod requestMethod,
Dictionary <string, IEnumerable <string> > requestHeaders,
Dictionary <string, IEnumerable <string> > contentHeaders,
IBlobSigner blobSigner,
IClock clock)
{
StorageClientImpl.ValidateBucketName(bucket);
var now = clock.GetCurrentDateTimeUtc();
var timestamp = now.ToString("yyyyMMdd'T'HHmmss'Z'", CultureInfo.InvariantCulture);
var datestamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
// TODO: Validate against maximum expiry duration
int expirySeconds = (int)(expiration - now).TotalSeconds;
string expiryText = expirySeconds.ToString(CultureInfo.InvariantCulture);
string credentialScope = $"{datestamp}/{DefaultRegion}/{ScopeSuffix}";
var headers = new SortedDictionary <string, string>(StringComparer.Ordinal);
headers["host"] = HostHeaderValue;
AddHeaders(headers, requestHeaders);
AddHeaders(headers, contentHeaders);
var canonicalHeaders = string.Join("", headers.Select(pair => $"{pair.Key}:{pair.Value}\n"));
var signedHeaders = string.Join(";", headers.Keys.Select(k => k.ToLowerInvariant()));
var queryParameters = new SortedDictionary <string, string>(StringComparer.Ordinal)
{
{ "X-Goog-Algorithm", Algorithm },
{ "X-Goog-Credential", $"{blobSigner.Id}/{credentialScope}" },
{ "X-Goog-Date", timestamp },
{ "X-Goog-Expires", expirySeconds.ToString(CultureInfo.InvariantCulture) },
{ "X-Goog-SignedHeaders", signedHeaders }
};
if (requestMethod == null)
{
requestMethod = HttpMethod.Get;
}
else if (requestMethod == ResumableHttpMethod)
{
requestMethod = HttpMethod.Post;
queryParameters["X-Goog-Resumable"] = "Start";
}
_canonicalQueryString = string.Join("&", queryParameters.Select(p => $"{p.Key}={Uri.EscapeDataString(p.Value)}"));
_resourcePath = $"/{bucket}";
if (objectName != null)
{
_resourcePath += $"/{Uri.EscapeDataString(objectName)}";
}
var canonicalRequest = $"{requestMethod}\n{_resourcePath}\n{_canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\nUNSIGNED-PAYLOAD";
string hashHex;
using (var sha256 = SHA256.Create())
{
hashHex = FormatHex(sha256.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest)));
}
_blobToSign = Encoding.UTF8.GetBytes($"{Algorithm}\n{timestamp}\n{credentialScope}\n{hashHex}");
void AddHeaders(SortedDictionary <string, string> canonicalized, IDictionary <string, IEnumerable <string> > headersToAdd)
{
if (headersToAdd == null)
{
return;
}
foreach (var pair in headersToAdd)
{
if (pair.Value == null)
{
continue;
}
var headerName = pair.Key.ToLowerInvariant();
if (headerName == EncryptionKey.KeyHeader || headerName == EncryptionKey.KeyHashHeader || headerName == EncryptionKey.AlgorithmHeader)
{
continue;
}
var value = string.Join(", ", pair.Value.Select(PrepareHeaderValue)).Trim();
if (canonicalized.TryGetValue(headerName, out var existingValue))
{
value = $"{existingValue}, {value}";
}
canonicalized[headerName] = value;
}
}
}
private static List <Timestamp> GenerateExpectedMetadata(IClock clock, params int[] timesInSecondsFromNow) => timesInSecondsFromNow .Select(t => clock.GetCurrentDateTimeUtc().AddSeconds(t)) .Select(Timestamp.FromDateTime) .ToList();