Esempio n. 1
0
 /// <inheritdoc />
 protected override void ReceiveWithSemaphoreHeld(IEnumerable <T> items)
 {
     GaxPreconditions.CheckNotNull(items, nameof(items));
     _items.AddRange(items);
     if (_clock.GetCurrentDateTimeUtc() >= _nextFlush)
     {
         FlushWithSemaphoreHeld();
     }
 }
Esempio n. 2
0
 public override Task WriteAsync(StreamingPullRequest message)
 {
     lock (_lock)
     {
         _extends.AddRange(message.ModifyDeadlineAckIds.Select(id => new TimedId(_clock.GetCurrentDateTimeUtc(), id)));
         _acks.AddRange(message.AckIds.Select(id => new TimedId(_clock.GetCurrentDateTimeUtc(), id)));
         return(Task.FromResult(0));
     }
 }
 public override async Task AcknowledgeAsync(SubscriptionName subscription, IEnumerable <string> ackIds, CancellationToken cancellationToken)
 {
     if (_writeAsyncPreDelay != TimeSpan.Zero)
     {
         await _taskHelper.ConfigureAwait(_scheduler.Delay(_writeAsyncPreDelay, CancellationToken.None));
     }
     lock (_lock)
     {
         _acks.AddRange(ackIds.Select(id => new TimedId(_clock.GetCurrentDateTimeUtc(), id)));
     }
 }
                public override async Task WriteAsync(StreamingPullRequest message)
                {
                    await _taskHelper.ConfigureAwait(_scheduler.Delay(_writeAsyncPreDelay, CancellationToken.None));

                    var now = _clock.GetCurrentDateTimeUtc();

                    lock (_lock)
                    {
                        _extends.AddRange(message.ModifyDeadlineAckIds.Select(id => new TimedId(now, id)).ToList());
                        _acks.AddRange(message.AckIds.Select(id => new TimedId(now, id)).ToList());
                    }
                }
Esempio n. 5
0
        // TODO: This is a modified/simplified version of ApiCallRetryExtensions.WithRetry from Google.Api.Gax.Grpc. Can we combine them somehow?
        internal static async Task RetryOperationUntilCompleted(
            Func <Task <bool> > fn,
            IClock clock,
            IScheduler scheduler,
            CallSettings callSettings,
            RetrySettings retrySettings)
        {
            DateTime?overallDeadline = retrySettings.TotalExpiration.CalculateDeadline(clock);
            TimeSpan retryDelay      = retrySettings.RetryBackoff.Delay;
            TimeSpan callTimeout     = retrySettings.TimeoutBackoff.Delay;

            while (true)
            {
                DateTime attemptDeadline = clock.GetCurrentDateTimeUtc() + callTimeout;
                // Note: this handles a null total deadline due to "<" returning false if overallDeadline is null.
                DateTime     combinedDeadline    = overallDeadline < attemptDeadline ? overallDeadline.Value : attemptDeadline;
                CallSettings attemptCallSettings = callSettings.WithCallTiming(CallTiming.FromDeadline(combinedDeadline));
                TimeSpan     actualDelay         = retrySettings.DelayJitter.GetDelay(retryDelay);
                DateTime     expectedRetryTime;
                try
                {
                    bool isResponseOk = await fn().ConfigureAwait(false);

                    if (isResponseOk)
                    {
                        return;
                    }

                    expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
                    if (expectedRetryTime > overallDeadline)
                    {
                        // TODO: Can we get this string from somewhere?
                        throw new RpcException(new Status(StatusCode.DeadlineExceeded, "Deadline Exceeded"));
                    }
                }
                catch (RpcException e) when(retrySettings.RetryFilter(e))
                {
                    expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
                    if (expectedRetryTime > overallDeadline)
                    {
                        throw;
                    }
                }
                await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);

                retryDelay  = retrySettings.RetryBackoff.NextDelay(retryDelay);
                callTimeout = retrySettings.TimeoutBackoff.NextDelay(callTimeout);
            }
        }
Esempio n. 6
0
        /// <summary>
        /// Asynchronously repeatedly calls the specified polling action, delaying between calls,
        /// until a given condition is met in the response.
        /// </summary>
        /// <typeparam name="TResponse">The response type.</typeparam>
        /// <param name="pollAction">The poll action, typically performing an RPC. The value passed to the
        /// action is the overall deadline, so that the RPC settings can be adjusted accordingly. A null
        /// value indicates no deadline.</param>
        /// <param name="completionPredicate">The test for whether to return the response (<c>true</c>) or continue
        /// polling (<c>false</c>). Must not be null.</param>
        /// <param name="clock">The clock to use for determining deadlines. Must not be null.</param>
        /// <param name="scheduler">The scheduler to use for delaying between calls. Must not be null.</param>
        /// <param name="pollSettings">The poll settings, controlling timeouts, call settings and delays.</param>
        /// <param name="cancellationToken">The cancellation token used to cancel delays, if any.</param>
        /// <returns>A task representing the asynchronous operation. The result of the task will be the completed response.</returns>
        public static async Task <TResponse> PollRepeatedlyAsync <TResponse>(
            Func <DateTime?, Task <TResponse> > pollAction,
            Predicate <TResponse> completionPredicate,
            IClock clock,
            IScheduler scheduler,
            PollSettings pollSettings,
            CancellationToken cancellationToken)
        {
            GaxPreconditions.CheckNotNull(pollAction, nameof(pollAction));
            GaxPreconditions.CheckNotNull(completionPredicate, nameof(completionPredicate));
            GaxPreconditions.CheckNotNull(clock, nameof(clock));
            GaxPreconditions.CheckNotNull(scheduler, nameof(scheduler));
            GaxPreconditions.CheckNotNull(pollSettings, nameof(pollSettings));

            var deadline     = pollSettings.Expiration.CalculateDeadline(clock);
            var currentDelay = pollSettings.Delay;

            while (true)
            {
                var latest = await pollAction(deadline).ConfigureAwait(false);

                if (completionPredicate(latest))
                {
                    return(latest);
                }
                if (clock.GetCurrentDateTimeUtc() + pollSettings.Delay >= deadline)
                {
                    // TODO: Could return null instead. Unclear what's better here.
                    throw new TimeoutException("Operation did not complete within the specified expiry time");
                }
                await scheduler.Delay(currentDelay, cancellationToken).ConfigureAwait(false);

                currentDelay = pollSettings.NextDelay(currentDelay);
            }
        }
 public override Task WriteAsync(StreamingPullRequest message)
 {
     if (message.ModifyDeadlineAckIds.Count != 0 || message.AckIds.Count != 0)
     {
         throw new InvalidOperationException("WriteAsync must not modify deadlines or send acks/nacks.");
     }
     if (Equals(message, new StreamingPullRequest()))
     {
         // An empty message is a ping
         lock (_lock)
         {
             _streamPings.Add(_clock.GetCurrentDateTimeUtc());
         }
     }
     return(Task.FromResult(0));
 }
Esempio n. 8
0
        /// <inheritdoc />
        public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
        {
            GaxPreconditions.CheckNotNull(formatter, nameof(formatter));

            if (!IsEnabled(logLevel))
            {
                return;
            }

            string message = formatter(state, exception);

            if (string.IsNullOrEmpty(message))
            {
                return;
            }

            LogEntry entry = new LogEntry
            {
                Resource    = _loggerOptions.MonitoredResource,
                LogName     = _logName,
                Severity    = logLevel.ToLogSeverity(),
                Timestamp   = Timestamp.FromDateTime(_clock.GetCurrentDateTimeUtc()),
                TextPayload = string.Concat(GoogleLoggerScope.Current, message),
                Labels      = { _loggerOptions.Labels },
                Trace       = GetTraceName() ?? "",
            };

            _consumer.Receive(new[] { entry });
        }
        internal async Task <TResult> ExecuteWithRetryAsync <TResult>(Func <Task <TResult> > fn, CancellationToken cancellationToken)
        {
            // Note: the way that we sometimes use the recommended retry delay, and base
            // further delays on that, means we can't use RetryAttempt easily here.
            DateTime?overallDeadline = _options.CalculateDeadline(_clock);
            TimeSpan retryDelay      = _options.InitialDelay;

            while (true)
            {
                try
                {
                    var result = await fn().ConfigureAwait(false);

                    return(result);
                }
                catch (SpannerException e) when(e.IsRetryable && !e.SessionExpired)
                {
                    // If there's a recommended retry delay specified on the exception
                    // we should respect it.
                    retryDelay = e.RecommendedRetryDelay ?? retryDelay;

                    TimeSpan actualDelay       = _options.Jitter(retryDelay);
                    DateTime expectedRetryTime = _clock.GetCurrentDateTimeUtc() + actualDelay;

                    if (expectedRetryTime > overallDeadline)
                    {
                        throw;
                    }
                    await _scheduler.Delay(actualDelay, cancellationToken).ConfigureAwait(false);

                    retryDelay = _options.NextDelay(retryDelay);
                }
            }
        }
Esempio n. 10
0
 private TimedBufferingConsumer(IConsumer <T> consumer, TimeSpan waitTime, IClock clock)
 {
     _consumer  = GaxPreconditions.CheckNotNull(consumer, nameof(consumer));
     _waitTime  = waitTime;
     _clock     = GaxPreconditions.CheckNotNull(clock, nameof(clock));
     _nextFlush = _clock.GetCurrentDateTimeUtc().Add(_waitTime);
 }
Esempio n. 11
0
        internal static async Task <TResponse> CallWithRetryAsync <TResponse>(
            this Func <Task <TResponse> > someMethod,
            CallSettings callSettings,
            IClock clock, IScheduler scheduler)
        {
            RetrySettings retrySettings = callSettings.Timing?.Retry;

            if (retrySettings == null)
            {
                return(await someMethod().ConfigureAwait(false));
            }
            DateTime?overallDeadline = retrySettings.TotalExpiration.CalculateDeadline(clock);
            TimeSpan retryDelay      = retrySettings.RetryBackoff.Delay;

            while (true)
            {
                try
                {
                    return(await someMethod().ConfigureAwait(false));
                }
                catch (RpcException e) when(retrySettings.RetryFilter(e))
                {
                    TimeSpan actualDelay       = retrySettings.DelayJitter.GetDelay(retryDelay);
                    DateTime expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;

                    if (expectedRetryTime > overallDeadline)
                    {
                        throw;
                    }
                    await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);

                    retryDelay = retrySettings.RetryBackoff.NextDelay(retryDelay);
                }
            }
        }
Esempio n. 12
0
        /// <inheritdoc />
        public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
        {
            try
            {
                GaxPreconditions.CheckNotNull(formatter, nameof(formatter));

                if (!IsEnabled(logLevel))
                {
                    return;
                }

                string message = formatter(state, exception);
                if (string.IsNullOrEmpty(message))
                {
                    return;
                }

                LogEntry entry = new LogEntry
                {
                    Resource    = _loggerOptions.MonitoredResource,
                    LogName     = _fullLogName,
                    Severity    = logLevel.ToLogSeverity(),
                    Timestamp   = Timestamp.FromDateTime(_clock.GetCurrentDateTimeUtc()),
                    JsonPayload = CreateJsonPayload(eventId, state, exception, message),
                    Labels      = { CreateLabels() },
                    Trace       = GetTraceName() ?? "",
                };

                _consumer.Receive(new[] { entry });
            }
            catch (Exception) when(_loggerOptions.RetryOptions.ExceptionHandling == ExceptionHandling.Ignore)
            {
            }
        }
Esempio n. 13
0
        /// <inheritdoc />
        public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
        {
            GaxPreconditions.CheckNotNull(formatter, nameof(formatter));

            if (!IsEnabled(logLevel))
            {
                return;
            }

            string message = formatter(state, exception);

            if (string.IsNullOrEmpty(message))
            {
                return;
            }

            LogEntry entry = new LogEntry
            {
                Resource    = _globalResource,
                LogName     = _logName,
                Severity    = logLevel.ToLogSeverity(),
                Timestamp   = Timestamp.FromDateTime(_clock.GetCurrentDateTimeUtc()),
                TextPayload = message,
            };

            _consumer.Receive(new[] { entry });
        }
Esempio n. 14
0
        // By design, the code is mostly duplicated between the async and sync calls.

        // Async retry
        internal static Func <TRequest, CallSettings, Task <TResponse> > WithRetry <TRequest, TResponse>(
            this Func <TRequest, CallSettings, Task <TResponse> > fn,
            IClock clock, IScheduler scheduler, Func <TResponse, Task> postResponse) =>
        async(request, callSettings) =>
        {
            RetrySettings retrySettings = callSettings.Timing?.Retry;
            if (retrySettings == null)
            {
                return(await fn(request, callSettings).ConfigureAwait(false));
            }
            DateTime?overallDeadline = retrySettings.TotalExpiration.CalculateDeadline(clock);
            TimeSpan retryDelay      = retrySettings.RetryBackoff.Delay;
            TimeSpan callTimeout     = retrySettings.TimeoutBackoff.Delay;
            while (true)
            {
                DateTime attemptDeadline = clock.GetCurrentDateTimeUtc() + callTimeout;
                // Note: this handles a null total deadline due to "<" returning false if overallDeadline is null.
                DateTime     combinedDeadline    = overallDeadline < attemptDeadline ? overallDeadline.Value : attemptDeadline;
                CallSettings attemptCallSettings = callSettings.WithCallTiming(CallTiming.FromDeadline(combinedDeadline));
                try
                {
                    var response = await fn(request, attemptCallSettings).ConfigureAwait(false);

                    if (postResponse != null)
                    {
                        await postResponse(response).ConfigureAwait(false);
                    }
                    return(response);
                }
                catch (RpcException e) when(retrySettings.RetryFilter(e))
                {
                    TimeSpan actualDelay       = retrySettings.DelayJitter.GetDelay(retryDelay);
                    DateTime expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;

                    if (expectedRetryTime > overallDeadline)
                    {
                        throw;
                    }
                    await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);

                    retryDelay  = retrySettings.RetryBackoff.NextDelay(retryDelay);
                    callTimeout = retrySettings.TimeoutBackoff.NextDelay(callTimeout);
                }
            }
        };
Esempio n. 15
0
                internal PostPolicySigningState(PostPolicy policy, Options options, IBlobSigner blobSigner, IClock clock)
                {
                    string uri = options.UrlStyle switch
                    {
                        UrlStyle.PathStyle => policy.Bucket == null ? StorageHost : $"{StorageHost}/{policy.Bucket}",
                        UrlStyle.VirtualHostedStyle => policy.Bucket == null ?
                        throw new ArgumentNullException(nameof(PostPolicy.Bucket), $"When using {UrlStyle.VirtualHostedStyle} a bucket condition must be set in the policy.") :
                              $"{policy.Bucket}.{StorageHost}",
                              UrlStyle.BucketBoundHostname => options.BucketBoundHostname,
                              _ => throw new ArgumentOutOfRangeException(nameof(options.UrlStyle))
                    };

                    uri = $"{options.Scheme}://{uri}/";

                    options = options.ToExpiration(clock);
                    var now = clock.GetCurrentDateTimeUtc();

                    int expirySeconds = (int)(options.Expiration.Value - now).TotalSeconds;

                    if (expirySeconds <= 0)
                    {
                        throw new ArgumentOutOfRangeException(nameof(options.Expiration), "Expiration must be at least 1 second.");
                    }
                    if (expirySeconds > MaxExpirySecondsInclusive)
                    {
                        throw new ArgumentOutOfRangeException(nameof(options.Expiration), "Expiration must not be greater than 7 days.");
                    }

                    var    datestamp       = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
                    string credentialScope = $"{datestamp}/{DefaultRegion}/{ScopeSuffix}";

                    policy.SetField(PolicyCreationDateTime.Element, new DateTimeOffset(now));
                    policy.SetField(PolicyAlgorithm.Element, Algorithm);
                    policy.SetField(PolicyCredential.Element, $"{blobSigner.Id}/{credentialScope}");

                    _expiration = options.Expiration.Value;
                    _policy     = policy;
                    _url        = new Uri(uri);

                    StringBuilder sb = new StringBuilder();
                    StringWriter  sw = new StringWriter(sb);

                    using (JsonWriter writer = new JsonTextWriter(sw))
                    {
                        writer.StringEscapeHandling = StringEscapeHandling.EscapeNonAscii;
                        writer.WriteStartObject();
                        policy.WriteTo(writer);
                        writer.WritePropertyName("expiration");
                        writer.WriteValue(_expiration.UtcDateTime.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'", CultureInfo.InvariantCulture));
                        writer.WriteEndObject();
                    }

                    var decodedPolicy = sb.ToString();

                    _encodedPolicy = Convert.ToBase64String(Encoding.UTF8.GetBytes(decodedPolicy));
                    _blobToSign    = Encoding.UTF8.GetBytes(_encodedPolicy);
                }
Esempio n. 16
0
        private LogEntry MakeLostEntry(DateTimeRange lostRange)
        {
            var lostEntry = _logsLostWarningTemplate.Clone();

            lostEntry.Timestamp   = _clock.GetCurrentDateTimeUtc().ToTimestamp();
            lostEntry.TextPayload = string.Format(lostEntry.TextPayload,
                                                  lostRange.From.ToString(GoogleStackdriverAppender.s_lostDateTimeFmt),
                                                  lostRange.To.ToString(GoogleStackdriverAppender.s_lostDateTimeFmt));
            return(lostEntry);
        }
        internal async Task RetryAsync(SpannerException abortedException, CancellationToken cancellationToken, int timeoutSeconds = MAX_TIMEOUT_SECONDS)
        {
            if (!EnableInternalRetries)
            {
                throw abortedException;
            }
            DateTime?overallDeadline = _options.CalculateDeadline(_clock);
            TimeSpan retryDelay      = _options.InitialDelay;

            // If there's a recommended retry delay specified on the exception
            // we should respect it.
            retryDelay = abortedException.RecommendedRetryDelay ?? _options.Jitter(retryDelay);
            while (true)
            {
                if (RetryCount >= MaxInternalRetryCount)
                {
                    throw new SpannerException(ErrorCode.Aborted, "Transaction was aborted because it aborted and retried too many times");
                }

                DateTime expectedRetryTime = _clock.GetCurrentDateTimeUtc() + retryDelay;
                if (expectedRetryTime > overallDeadline)
                {
                    throw new SpannerException(ErrorCode.Aborted, "Transaction was aborted because it timed out while retrying");
                }
                await _scheduler.Delay(retryDelay, cancellationToken).ConfigureAwait(false);

                // TODO: Preferably the Spanner client library should have some 'reset' option on an existing
                // transaction instead of having to begin a new transaction. This will potentially use a transaction
                // on a different session, while the recommended behavior for a retry is to use the same session as
                // the original attempt.
                SpannerTransaction.Dispose();
                SpannerTransaction = await Connection.SpannerConnection.BeginTransactionAsync(cancellationToken);

                RetryCount++;
                try
                {
                    foreach (IRetriableStatement statement in _retriableStatements)
                    {
                        await statement.RetryAsync(this, cancellationToken, timeoutSeconds);
                    }
                    break;
                }
                catch (SpannerAbortedDueToConcurrentModificationException)
                {
                    // Retry failed because of a concurrent modification.
                    throw;
                }
                catch (SpannerException e) when(e.ErrorCode == ErrorCode.Aborted)
                {
                    // Ignore and retry.
                    retryDelay = e.RecommendedRetryDelay ?? _options.Jitter(_options.NextDelay(retryDelay));
                }
            }
        }
 /// <inheritdoc />
 public void Receive(IEnumerable <T> items)
 {
     GaxPreconditions.CheckNotNull(items, nameof(items));
     lock (_mutex)
     {
         _items.AddRange(items);
         if (_clock.GetCurrentDateTimeUtc() >= _nextFlush)
         {
             Flush();
         }
     }
 }
Esempio n. 19
0
            /// <summary>
            /// Updates the state on the basis of the given exception, delaying for as long as is necessary
            /// between retries.
            /// </summary>
            internal async Task WaitAsync(RpcException exception, CancellationToken cancellationToken)
            {
                TimeSpan?delayFromException = GetRetryDelay(exception);
                TimeSpan delay = _retrySettings.BackoffJitter.GetDelay(delayFromException ?? _retrySettingsBackoffs.Current);

                if (_clock.GetCurrentDateTimeUtc() + delay >= _currentDeadline)
                {
                    throw new RpcException(new Grpc.Core.Status(StatusCode.DeadlineExceeded, "Streaming call exceeded timeout"));
                }
                await _scheduler.Delay(delay, cancellationToken).ConfigureAwait(false);

                if (delayFromException == null)
                {
                    _retrySettingsBackoffs.MoveNext();
                }
            }
        // By design, the code is mostly duplicated between the async and sync calls.

        // Async retry
        internal static Func <TRequest, CallSettings, Task <TResponse> > WithRetry <TRequest, TResponse>(
            this Func <TRequest, CallSettings, Task <TResponse> > fn,
            IClock clock, IScheduler scheduler, Func <TResponse, Task> postResponse) =>
        async(request, callSettings) =>
        {
            RetrySettings retrySettings = callSettings.Retry;
            if (retrySettings == null)
            {
                return(await fn(request, callSettings).ConfigureAwait(false));
            }
            DateTime?overallDeadline = callSettings.Expiration.CalculateDeadline(clock);
            // Every attempt should use the same deadline, calculated from the start of the call.
            if (callSettings.Expiration?.Type == ExpirationType.Timeout)
            {
                callSettings = callSettings.WithDeadline(overallDeadline.Value);
            }
            TimeSpan backoffDelay = retrySettings.InitialBackoff;
            int      attempt      = 0;
            while (true)
            {
                try
                {
                    attempt++;
                    var response = await fn(request, callSettings).ConfigureAwait(false);

                    if (postResponse != null)
                    {
                        await postResponse(response).ConfigureAwait(false);
                    }
                    return(response);
                }
                catch (RpcException e) when(attempt < retrySettings.MaxAttempts && retrySettings.RetryFilter(e))
                {
                    TimeSpan actualDelay       = retrySettings.BackoffJitter.GetDelay(backoffDelay);
                    DateTime expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;

                    if (expectedRetryTime > overallDeadline)
                    {
                        throw;
                    }
                    await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);

                    backoffDelay = retrySettings.NextBackoff(backoffDelay);
                }
            }
        };
Esempio n. 21
0
        private void ActivateLogIdAndResource()
        {
            string            projectId = null;
            MonitoredResource resource  = null;

            if (!DisableResourceTypeDetection)
            {
                resource = MonitoredResourceBuilder.FromPlatform(_platform);
                resource.Labels.TryGetValue("project_id", out projectId);
            }
            if (projectId == null)
            {
                // Either platform detection is disabled, or it detected an unknown platform
                // So use the manually configured projectId and override the resource
                projectId = GaxPreconditions.CheckNotNull(ProjectId, nameof(ProjectId));
                if (ResourceType == null)
                {
                    resource = new MonitoredResource {
                        Type = "global", Labels = { { "project_id", projectId } }
                    };
                }
                else
                {
                    resource = new MonitoredResource {
                        Type = ResourceType, Labels = { _resourceLabels.ToDictionary(x => x.Key, x => x.Value) }
                    };
                }
            }
            else
            {
                if (ProjectId != null && projectId != ProjectId)
                {
                    _preLogs.Add(() => BuildLogEntry(new LoggingEvent(new LoggingEventData
                    {
                        Level      = Level.Warn,
                        LoggerName = "",
                        Message    = s_mismatchedProjectIdMessage,
                        TimeStamp  = _clock.GetCurrentDateTimeUtc()
                    })));
                }
            }
            _resource = resource;
            _logName  = new LogName(projectId, LogId).ToString();
        }
Esempio n. 22
0
        /// <summary>
        /// Calculate a deadline from an <see cref="Expiration"/> and a <see cref="IClock"/>.
        /// </summary>
        /// <param name="expiration"><see cref="Expiration"/>, may be null.</param>
        /// <param name="clock"><see cref="IClock"/> to use for deadline calculation.</param>
        /// <returns>The calculated absolute deadline, or null if no deadline should be used.</returns>
        internal static DateTime?CalculateDeadline(this Expiration expiration, IClock clock)
        {
            GaxPreconditions.CheckNotNull(clock, nameof(clock));
            if (expiration == null)
            {
                return(null);
            }
            switch (expiration.Type)
            {
            case ExpirationType.None:
                return(null);

            case ExpirationType.Timeout:
                return(clock.GetCurrentDateTimeUtc() + expiration.Timeout.Value);

            case ExpirationType.Deadline:
                return(expiration.Deadline.Value);

            default:
                throw new ArgumentException("Invalid expiration", nameof(expiration));
            }
        }
        /// <summary>
        /// Extracts the cache value from a Firestore document snapshot.
        /// Checks all the expiration fields so never returns a stale value.
        /// </summary>
        /// <param name="snapshot">The snapshot to pull the cache value from.</param>
        /// <returns>The cache value or null.</returns>
        private byte[] ValueFromSnapshot(DocumentSnapshot snapshot)
        {
            if (!snapshot.Exists)
            {
                return(null);
            }
            CacheDoc doc = snapshot.ConvertTo <CacheDoc>();
            var      now = _clock.GetCurrentDateTimeUtc();

            if (doc.AbsoluteExpiration < now)
            {
                return(null);
            }
            var slidingExpiration = doc.LastRefresh.GetValueOrDefault()
                                    + TimeSpan.FromSeconds(doc.SlidingExpirationSeconds.GetValueOrDefault());

            if (slidingExpiration < now)
            {
                return(null);
            }
            return(doc.Value);
        }
Esempio n. 24
0
        /// <inheritdoc />
        public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
        {
            try
            {
                GaxPreconditions.CheckNotNull(formatter, nameof(formatter));

                if (!IsEnabled(logLevel))
                {
                    return;
                }

                string message = formatter(state, exception);
                if (string.IsNullOrEmpty(message))
                {
                    return;
                }

                var jsonStruct = new Struct();
                jsonStruct.Fields.Add("message", Value.ForString(message));
                jsonStruct.Fields.Add("log_name", Value.ForString(_logName));

                if (eventId.Id != 0 || eventId.Name != null)
                {
                    var eventStruct = new Struct();
                    if (eventId.Id != 0)
                    {
                        eventStruct.Fields.Add("id", Value.ForNumber(eventId.Id));
                    }
                    if (!string.IsNullOrWhiteSpace(eventId.Name))
                    {
                        eventStruct.Fields.Add("name", Value.ForString(eventId.Name));
                    }
                    jsonStruct.Fields.Add("event_id", Value.ForStruct(eventStruct));
                }

                // If we have format params and its more than just the original message add them.
                if (state is IEnumerable <KeyValuePair <string, object> > formatParams &&
                    !(formatParams.Count() == 1 && formatParams.Single().Key.Equals("{OriginalFormat}")))
                {
                    var paramStruct = new Struct();
                    foreach (var pair in formatParams)
                    {
                        // Consider adding formatting support for values that are IFormattable.
                        paramStruct.Fields[pair.Key] = Value.ForString(pair.Value?.ToString() ?? "");
                    }

                    if (paramStruct.Fields.Count > 0)
                    {
                        jsonStruct.Fields.Add("format_parameters", Value.ForStruct(paramStruct));
                    }
                }

                var currentLogScope = GoogleLoggerScope.Current;
                if (currentLogScope != null)
                {
                    jsonStruct.Fields.Add("scope", Value.ForString(currentLogScope.ToString()));
                }

                // Create a map of format parameters of all the parent scopes,
                // starting from the most inner scope to the top-level scope.
                var scopeParamsList = new List <Value>();
                while (currentLogScope != null)
                {
                    // Determine if the state of the scope are format params
                    if (currentLogScope.State is FormattedLogValues scopeFormatParams)
                    {
                        var scopeParams = new Struct();
                        foreach (var pair in scopeFormatParams)
                        {
                            scopeParams.Fields[pair.Key] = Value.ForString(pair.Value?.ToString() ?? "");
                        }

                        scopeParamsList.Add(Value.ForStruct(scopeParams));
                    }

                    currentLogScope = currentLogScope.Parent;
                }

                if (scopeParamsList.Count > 0)
                {
                    jsonStruct.Fields.Add("parent_scopes", Value.ForList(scopeParamsList.ToArray()));
                }

                Dictionary <string, string> labels;
                var labelProviders = GetLabelProviders()?.ToArray();
                if (labelProviders?.Length > 0)
                {
                    // Create a copy of the labels from the options and invoke each provider
                    labels = _loggerOptions.Labels.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
                    foreach (var provider in labelProviders)
                    {
                        provider.Invoke(labels);
                    }
                }
                else
                {
                    labels = _loggerOptions.Labels;
                }

                LogEntry entry = new LogEntry
                {
                    Resource    = _loggerOptions.MonitoredResource,
                    LogName     = _fullLogName,
                    Severity    = logLevel.ToLogSeverity(),
                    Timestamp   = Timestamp.FromDateTime(_clock.GetCurrentDateTimeUtc()),
                    JsonPayload = jsonStruct,
                    Labels      = { labels },
                    Trace       = GetTraceName() ?? "",
                };

                _consumer.Receive(new[] { entry });
            }
            catch (Exception) when(_loggerOptions.RetryOptions.ExceptionHandling == ExceptionHandling.Ignore)
            {
            }
        }
 /// <summary>
 /// If this set of options was duration based, this method will return a new set
 /// of options whose expiration will be calculated based on this instance duration
 /// and the given clock.
 /// If this set of options was expiration based this same instance will be returned.
 /// </summary>
 /// <param name="clock">The clock to use to calculate the expiration.</param>
 /// <returns>An expiration based set of options.</returns>
 internal Options ToExpiration(IClock clock) =>
 Expiration.HasValue ? this : WithExpiration(clock.GetCurrentDateTimeUtc() + Duration.Value);
Esempio n. 26
0
                internal UrlSigningState(RequestTemplate template, Options options, IBlobSigner blobSigner, IClock clock)
                {
                    (_host, _resourcePath) = options.UrlStyle switch
                    {
                        UrlStyle.PathStyle => (StorageHost, $"/{template.Bucket}"),
                        UrlStyle.VirtualHostedStyle => ($"{template.Bucket}.{StorageHost}", string.Empty),
                        UrlStyle.BucketBoundHostname => (options.BucketBoundHostname, string.Empty),
                        _ => throw new ArgumentOutOfRangeException(nameof(options.UrlStyle))
                    };

                    _scheme = options.Scheme;
                    options = options.ToExpiration(clock);

                    var now           = clock.GetCurrentDateTimeUtc();
                    var timestamp     = now.ToString("yyyyMMdd'T'HHmmss'Z'", CultureInfo.InvariantCulture);
                    var datestamp     = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
                    int expirySeconds = (int)(options.Expiration.Value - now).TotalSeconds;

                    if (expirySeconds <= 0)
                    {
                        throw new ArgumentOutOfRangeException(nameof(options.Expiration), "Expiration must be at least 1 second");
                    }
                    if (expirySeconds > MaxExpirySecondsInclusive)
                    {
                        throw new ArgumentOutOfRangeException(nameof(options.Expiration), "Expiration must not be greater than 7 days.");
                    }

                    string expiryText = expirySeconds.ToString(CultureInfo.InvariantCulture);

                    string credentialScope = $"{datestamp}/{DefaultRegion}/{ScopeSuffix}";

                    var headers = new SortedDictionary <string, string>(StringComparer.Ordinal);

                    headers.AddHeader("host", _host);
                    headers.AddHeaders(template.RequestHeaders);
                    headers.AddHeaders(template.ContentHeaders);
                    var canonicalHeaders = string.Join("", headers.Select(pair => $"{pair.Key}:{pair.Value}\n"));
                    var signedHeaders    = string.Join(";", headers.Keys.Select(k => k.ToLowerInvariant()));

                    var queryParameters = new SortedSet <string>(StringComparer.Ordinal);

                    queryParameters.AddQueryParameter("X-Goog-Algorithm", Algorithm);
                    queryParameters.AddQueryParameter("X-Goog-Credential", $"{blobSigner.Id}/{credentialScope}");
                    queryParameters.AddQueryParameter("X-Goog-Date", timestamp);
                    queryParameters.AddQueryParameter("X-Goog-Expires", expirySeconds.ToString(CultureInfo.InvariantCulture));
                    queryParameters.AddQueryParameter("X-Goog-SignedHeaders", signedHeaders);

                    var effectiveRequestMethod = template.HttpMethod;

                    if (effectiveRequestMethod == ResumableHttpMethod)
                    {
                        effectiveRequestMethod = HttpMethod.Post;
                        queryParameters.AddQueryParameter("X-Goog-Resumable", "Start");
                    }

                    queryParameters.AddQueryParameters(template.QueryParameters);

                    _canonicalQueryString = string.Join("&", queryParameters);
                    if (!string.IsNullOrEmpty(template.ObjectName))
                    {
                        // EscapeDataString escapes slashes, which we *don't* want to escape here. The simplest option is to
                        // split the path into segments by slashes, escape each segment, then join the escaped segments together again.
                        var segments = template.ObjectName.Split('/');
                        var escaped  = string.Join("/", segments.Select(Uri.EscapeDataString));
                        _resourcePath = _resourcePath + "/" + escaped;
                    }

                    string payloadHash       = "UNSIGNED-PAYLOAD";
                    var    payloadHashHeader = headers.Where(
                        header => header.Key.Equals("X-Goog-Content-SHA256", StringComparison.OrdinalIgnoreCase)).ToList();

                    if (payloadHashHeader.Count == 1)
                    {
                        payloadHash = payloadHashHeader[0].Value;
                    }

                    var canonicalRequest = $"{effectiveRequestMethod}\n{_resourcePath}\n{_canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\n{payloadHash}";

                    string hashHex;

                    using (var sha256 = SHA256.Create())
                    {
                        hashHex = FormatHex(sha256.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest)));
                    }

                    _blobToSign = Encoding.UTF8.GetBytes($"{Algorithm}\n{timestamp}\n{credentialScope}\n{hashHex}");
                }
                internal SigningState(
                    string bucket,
                    string objectName,
                    DateTimeOffset expiration,
                    HttpMethod requestMethod,
                    Dictionary <string, IEnumerable <string> > requestHeaders,
                    Dictionary <string, IEnumerable <string> > contentHeaders,
                    IBlobSigner blobSigner,
                    IClock clock)
                {
                    StorageClientImpl.ValidateBucketName(bucket);

                    var now           = clock.GetCurrentDateTimeUtc();
                    var timestamp     = now.ToString("yyyyMMdd'T'HHmmss'Z'", CultureInfo.InvariantCulture);
                    var datestamp     = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
                    int expirySeconds = (int)(expiration - now).TotalSeconds;

                    if (expirySeconds <= 0)
                    {
                        throw new ArgumentOutOfRangeException(nameof(expiration), "Expiration must be at least 1 second");
                    }
                    if (expirySeconds > MaxExpirySecondsInclusive)
                    {
                        throw new ArgumentOutOfRangeException(nameof(expiration), "Expiration must not be greater than 7 days.");
                    }

                    string expiryText = expirySeconds.ToString(CultureInfo.InvariantCulture);

                    string credentialScope = $"{datestamp}/{DefaultRegion}/{ScopeSuffix}";

                    var headers = new SortedDictionary <string, string>(StringComparer.Ordinal);

                    headers["host"] = HostHeaderValue;
                    AddHeaders(headers, requestHeaders);
                    AddHeaders(headers, contentHeaders);
                    var canonicalHeaders = string.Join("", headers.Select(pair => $"{pair.Key}:{pair.Value}\n"));
                    var signedHeaders    = string.Join(";", headers.Keys.Select(k => k.ToLowerInvariant()));

                    var queryParameters = new SortedDictionary <string, string>(StringComparer.Ordinal)
                    {
                        { "X-Goog-Algorithm", Algorithm },
                        { "X-Goog-Credential", $"{blobSigner.Id}/{credentialScope}" },
                        { "X-Goog-Date", timestamp },
                        { "X-Goog-Expires", expirySeconds.ToString(CultureInfo.InvariantCulture) },
                        { "X-Goog-SignedHeaders", signedHeaders }
                    };

                    if (requestMethod == null)
                    {
                        requestMethod = HttpMethod.Get;
                    }
                    else if (requestMethod == ResumableHttpMethod)
                    {
                        requestMethod = HttpMethod.Post;
                        queryParameters["X-Goog-Resumable"] = "Start";
                    }

                    _canonicalQueryString = string.Join("&", queryParameters.Select(p => $"{p.Key}={Uri.EscapeDataString(p.Value)}"));
                    _resourcePath         = $"/{bucket}";
                    if (!string.IsNullOrEmpty(objectName))
                    {
                        // EscapeDataString escapes slashes, which we *don't* want to escape here. The simplest option is to
                        // split the path into segments by slashes, escape each segment, then join the escaped segments together again.
                        var segments = objectName.Split('/');
                        var escaped  = string.Join("/", segments.Select(Uri.EscapeDataString));
                        _resourcePath = _resourcePath + "/" + escaped;
                    }

                    var canonicalRequest = $"{requestMethod}\n{_resourcePath}\n{_canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\nUNSIGNED-PAYLOAD";

                    string hashHex;

                    using (var sha256 = SHA256.Create())
                    {
                        hashHex = FormatHex(sha256.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest)));
                    }

                    _blobToSign = Encoding.UTF8.GetBytes($"{Algorithm}\n{timestamp}\n{credentialScope}\n{hashHex}");

                    void AddHeaders(SortedDictionary <string, string> canonicalized, IDictionary <string, IEnumerable <string> > headersToAdd)
                    {
                        if (headersToAdd == null)
                        {
                            return;
                        }
                        foreach (var pair in headersToAdd)
                        {
                            if (pair.Value == null)
                            {
                                continue;
                            }
                            var headerName = pair.Key.ToLowerInvariant();
                            // Note: the comma-space separating here is because this is what HttpClient does.
                            // Google Cloud Storage itself will just use commas if it receives multiple values for the same header name,
                            // but HttpClient coalesces the values itself. This approach means that if the same request is made from .NET
                            // with the signed URL, it will succeed - but it does mean that the signed URL won't be valid when used from
                            // another platform that sends actual multiple values.
                            var value = string.Join(", ", pair.Value.Select(PrepareHeaderValue)).Trim();
                            if (canonicalized.TryGetValue(headerName, out var existingValue))
                            {
                                value = $"{existingValue}, {value}";
                            }
                            canonicalized[headerName] = value;
                        }
                    }
                }
Esempio n. 28
0
 /// <summary>
 /// Calculate a deadline from an <see cref="Expiration"/> and a <see cref="IClock"/>.
 /// </summary>
 /// <param name="expiration"><see cref="Expiration"/>, may be null.</param>
 /// <param name="clock"><see cref="IClock"/> to use for deadline calculation.</param>
 /// <returns>The calculated absolute deadline, or null if no deadline should be used.</returns>
 internal static DateTime? CalculateDeadline(this Expiration expiration, IClock clock)
 {
     GaxPreconditions.CheckNotNull(clock, nameof(clock));
     if (expiration == null)
     {
         return null;
     }
     switch (expiration.Type)
     {
         case ExpirationType.None:
             return null;
         case ExpirationType.Timeout:
             return clock.GetCurrentDateTimeUtc() + expiration.Timeout.Value;
         case ExpirationType.Deadline:
             return expiration.Deadline.Value;
         default:
             throw new ArgumentException("Invalid expiration", nameof(expiration));
     }
 }
                internal SigningState(
                    string bucket,
                    string objectName,
                    DateTimeOffset expiration,
                    HttpMethod requestMethod,
                    Dictionary <string, IEnumerable <string> > requestHeaders,
                    Dictionary <string, IEnumerable <string> > contentHeaders,
                    IBlobSigner blobSigner,
                    IClock clock)
                {
                    StorageClientImpl.ValidateBucketName(bucket);

                    bool isResumableUpload = false;

                    if (requestMethod == null)
                    {
                        requestMethod = HttpMethod.Get;
                    }
                    else if (requestMethod == ResumableHttpMethod)
                    {
                        isResumableUpload = true;
                        requestMethod     = HttpMethod.Post;
                    }

                    var now       = clock.GetCurrentDateTimeUtc();
                    var timestamp = now.ToString("yyyyMMdd'T'HHmmss'Z'", CultureInfo.InvariantCulture);
                    var datestamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
                    // TODO: Validate again maximum expirary duration
                    int    expirySeconds = (int)(expiration - now).TotalSeconds;
                    string expiryText    = expirySeconds.ToString(CultureInfo.InvariantCulture);

                    string clientEmail     = blobSigner.Id;
                    string credentialScope = $"{datestamp}/auto/gcs/goog4_request";
                    string credential      = WebUtility.UrlEncode($"{blobSigner.Id}/{credentialScope}");

                    // FIXME: Use requestHeaders and contentHeaders
                    var headers = new SortedDictionary <string, string>();

                    headers["host"] = "storage.googleapis.com";

                    var canonicalHeaderBuilder = new StringBuilder();

                    foreach (var pair in headers)
                    {
                        canonicalHeaderBuilder.Append($"{pair.Key}:{pair.Value}\n");
                    }

                    var canonicalHeaders = canonicalHeaderBuilder.ToString().ToLowerInvariant();
                    var signedHeaders    = string.Join(";", headers.Keys.Select(k => k.ToLowerInvariant()));

                    queryParameters = new List <string>
                    {
                        "X-Goog-Algorithm=GOOG4-RSA-SHA256",
                        $"X-Goog-Credential={credential}",
                        $"X-Goog-Date={timestamp}",
                        $"X-Goog-Expires={expirySeconds}",
                        $"X-Goog-SignedHeaders={signedHeaders}"
                    };
                    if (isResumableUpload)
                    {
                        queryParameters.Insert(4, "X-Goog-Resumable=Start");
                    }

                    var canonicalQueryString = string.Join("&", queryParameters);

                    resourcePath = $"/{bucket}";
                    if (objectName != null)
                    {
                        resourcePath += $"/{Uri.EscapeDataString(objectName)}";
                    }

                    var    canonicalRequest = $"{requestMethod}\n{resourcePath}\n{canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\nUNSIGNED-PAYLOAD";
                    string hashHex;

                    using (var sha256 = SHA256.Create())
                    {
                        hashHex = FormatHex(sha256.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest)));
                    }

                    blobToSign = Encoding.UTF8.GetBytes($"GOOG4-RSA-SHA256\n{timestamp}\n{credentialScope}\n{hashHex}");
                }
                internal SigningState(
                    string bucket,
                    string objectName,
                    DateTimeOffset expiration,
                    HttpMethod requestMethod,
                    Dictionary <string, IEnumerable <string> > requestHeaders,
                    Dictionary <string, IEnumerable <string> > contentHeaders,
                    IBlobSigner blobSigner,
                    IClock clock)
                {
                    StorageClientImpl.ValidateBucketName(bucket);

                    var now       = clock.GetCurrentDateTimeUtc();
                    var timestamp = now.ToString("yyyyMMdd'T'HHmmss'Z'", CultureInfo.InvariantCulture);
                    var datestamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
                    // TODO: Validate against maximum expiry duration
                    int    expirySeconds = (int)(expiration - now).TotalSeconds;
                    string expiryText    = expirySeconds.ToString(CultureInfo.InvariantCulture);

                    string credentialScope = $"{datestamp}/{DefaultRegion}/{ScopeSuffix}";

                    var headers = new SortedDictionary <string, string>(StringComparer.Ordinal);

                    headers["host"] = HostHeaderValue;
                    AddHeaders(headers, requestHeaders);
                    AddHeaders(headers, contentHeaders);
                    var canonicalHeaders = string.Join("", headers.Select(pair => $"{pair.Key}:{pair.Value}\n"));
                    var signedHeaders    = string.Join(";", headers.Keys.Select(k => k.ToLowerInvariant()));

                    var queryParameters = new SortedDictionary <string, string>(StringComparer.Ordinal)
                    {
                        { "X-Goog-Algorithm", Algorithm },
                        { "X-Goog-Credential", $"{blobSigner.Id}/{credentialScope}" },
                        { "X-Goog-Date", timestamp },
                        { "X-Goog-Expires", expirySeconds.ToString(CultureInfo.InvariantCulture) },
                        { "X-Goog-SignedHeaders", signedHeaders }
                    };

                    if (requestMethod == null)
                    {
                        requestMethod = HttpMethod.Get;
                    }
                    else if (requestMethod == ResumableHttpMethod)
                    {
                        requestMethod = HttpMethod.Post;
                        queryParameters["X-Goog-Resumable"] = "Start";
                    }

                    _canonicalQueryString = string.Join("&", queryParameters.Select(p => $"{p.Key}={Uri.EscapeDataString(p.Value)}"));
                    _resourcePath         = $"/{bucket}";
                    if (objectName != null)
                    {
                        _resourcePath += $"/{Uri.EscapeDataString(objectName)}";
                    }

                    var canonicalRequest = $"{requestMethod}\n{_resourcePath}\n{_canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\nUNSIGNED-PAYLOAD";

                    string hashHex;

                    using (var sha256 = SHA256.Create())
                    {
                        hashHex = FormatHex(sha256.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest)));
                    }

                    _blobToSign = Encoding.UTF8.GetBytes($"{Algorithm}\n{timestamp}\n{credentialScope}\n{hashHex}");

                    void AddHeaders(SortedDictionary <string, string> canonicalized, IDictionary <string, IEnumerable <string> > headersToAdd)
                    {
                        if (headersToAdd == null)
                        {
                            return;
                        }
                        foreach (var pair in headersToAdd)
                        {
                            if (pair.Value == null)
                            {
                                continue;
                            }
                            var headerName = pair.Key.ToLowerInvariant();
                            if (headerName == EncryptionKey.KeyHeader || headerName == EncryptionKey.KeyHashHeader || headerName == EncryptionKey.AlgorithmHeader)
                            {
                                continue;
                            }
                            var value = string.Join(", ", pair.Value.Select(PrepareHeaderValue)).Trim();
                            if (canonicalized.TryGetValue(headerName, out var existingValue))
                            {
                                value = $"{existingValue}, {value}";
                            }
                            canonicalized[headerName] = value;
                        }
                    }
                }
Esempio n. 31
0
 private static List <Timestamp> GenerateExpectedMetadata(IClock clock, params int[] timesInSecondsFromNow) =>
 timesInSecondsFromNow
 .Select(t => clock.GetCurrentDateTimeUtc().AddSeconds(t))
 .Select(Timestamp.FromDateTime)
 .ToList();