Пример #1
0
 public BackOffController(ICounter counter, IBackOff dac, ITime time, IOTP otp)
 {
     this.counter = counter;
     this.dac     = dac;
     this.time    = time;
     this.otp     = otp;
 }
Пример #2
0
 /// <summary>Constructs a new initializer by the given back-off.</summary>
 public Initializer(IBackOff backOff)
 {
     BackOff = backOff;
     HandleExceptionFunc = DefaultHandleExceptionFunc;
     HandleUnsuccessfulResponseFunc = DefaultHandleUnsuccessfulResponseFunc;
     MaxTimeSpan = TimeSpan.FromSeconds(16);
 }
Пример #3
0
 /// <summary>Constructs a new initializer by the given back-off.</summary>
 public Initializer(IBackOff backOff)
 {
     BackOff                        = backOff;
     HandleExceptionFunc            = DefaultHandleExceptionFunc;
     HandleUnsuccessfulResponseFunc = DefaultHandleUnsuccessfulResponseFunc;
     MaxTimeSpan                    = TimeSpan.FromSeconds(16);
 }
Пример #4
0
 public Synchronizer(ISplitFetcher splitFetcher,
                     ISelfRefreshingSegmentFetcher segmentFetcher,
                     IImpressionsLog impressionsLog,
                     IEventsLog eventsLog,
                     IImpressionsCountSender impressionsCountSender,
                     IWrapperAdapter wrapperAdapter,
                     IStatusManager statusManager,
                     ITelemetrySyncTask telemetrySyncTask,
                     ITasksManager tasksManager,
                     ISplitCache splitCache,
                     IBackOff backOff,
                     int onDemandFetchMaxRetries,
                     int onDemandFetchRetryDelayMs,
                     ISegmentCache segmentCache,
                     ISplitLogger log = null)
 {
     _splitFetcher              = splitFetcher;
     _segmentFetcher            = segmentFetcher;
     _impressionsLog            = impressionsLog;
     _eventsLog                 = eventsLog;
     _impressionsCountSender    = impressionsCountSender;
     _wrapperAdapter            = wrapperAdapter;
     _statusManager             = statusManager;
     _telemetrySyncTask         = telemetrySyncTask;
     _tasksManager              = tasksManager;
     _splitCache                = splitCache;
     _backOffSplits             = backOff;
     _backOffSegments           = backOff;
     _onDemandFetchMaxRetries   = onDemandFetchMaxRetries;
     _onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs;
     _segmentCache              = segmentCache;
     _log = log ?? WrapperAdapter.GetLogger(typeof(Synchronizer));
     _defaultFetchOptions = new FetchOptions();
 }
Пример #5
0
 public PushManager(int authRetryBackOffBase,
                    ISSEHandler sseHandler,
                    IAuthApiClient authApiClient,
                    IWrapperAdapter wrapperAdapter = null,
                    ISplitLogger log = null,
                    IBackOff backOff = null)
 {
     _sseHandler     = sseHandler;
     _authApiClient  = authApiClient;
     _log            = log ?? WrapperAdapter.GetLogger(typeof(PushManager));
     _wrapperAdapter = wrapperAdapter ?? new WrapperAdapter();
     _backOff        = backOff ?? new BackOff(authRetryBackOffBase, attempt: 1);
 }
Пример #6
0
        public EventSourceClient(int backOffBase,
                                 ISplitLogger log = null,
                                 INotificationParser notificationParser = null,
                                 IBackOff backOff = null,
                                 IWrapperAdapter wrapperAdapter = null)
        {
            _log = log ?? WrapperAdapter.GetLogger(typeof(EventSourceClient));
            _notificationParser = notificationParser ?? new NotificationParser();
            _backOff            = backOff ?? new BackOff(backOffBase);
            _wrapperAdapter     = wrapperAdapter ?? new WrapperAdapter();

            UpdateFinishedConnection(finished: true);
        }
Пример #7
0
 public PushManager(ISSEHandler sseHandler,
                    IAuthApiClient authApiClient,
                    IWrapperAdapter wrapperAdapter,
                    ITelemetryRuntimeProducer telemetryRuntimeProducer,
                    IBackOff backOff,
                    ISplitLogger log = null)
 {
     _sseHandler               = sseHandler;
     _authApiClient            = authApiClient;
     _log                      = log ?? WrapperAdapter.GetLogger(typeof(PushManager));
     _wrapperAdapter           = wrapperAdapter;
     _backOff                  = backOff;
     _telemetryRuntimeProducer = telemetryRuntimeProducer;
 }
Пример #8
0
        private void Initialize(BigQueryOutputConfiguration bqOutputConfiguration)
        {
            Debug.Assert(bqOutputConfiguration != null);
            Debug.Assert(this.healthReporter != null);

            if (String.IsNullOrEmpty(bqOutputConfiguration.ProjectId))
            {
                throw new ArgumentException("ProjectId");
            }
            if (String.IsNullOrEmpty(bqOutputConfiguration.DatasetId))
            {
                throw new ArgumentException("DatasetId");
            }
            if (String.IsNullOrEmpty(bqOutputConfiguration.TableId))
            {
                throw new ArgumentException("TableId");
            }

            this.Config = bqOutputConfiguration;

            // Load table schema file
            if (Config.TableSchemaFile != null)
            {
                TableSchema = LoadTableSchema(Config.TableSchemaFile);
            }
            if (TableSchema == null)
            {
                throw new Exception("table schema not set");
            }
            // Expand table id 1st time within force mode
            ExpandTableIdIfNecessary(force: true);
            // configure finished
            healthReporter.ReportHealthy("TableId: " + TableIdExpanded);

            var scopes = new[]
            {
                BigqueryService.Scope.Bigquery,
                BigqueryService.Scope.BigqueryInsertdata,
                BigqueryService.Scope.CloudPlatform,
                BigqueryService.Scope.DevstorageFullControl
            };
            var credential = GoogleCredential.GetApplicationDefault()
                             .CreateScoped(scopes);

            _BQSvc = new BigqueryService(new BaseClientService.Initializer
            {
                HttpClientInitializer = credential,
            });
            _BackOff = new ExponentialBackOff();
        }
Пример #9
0
 /// <summary>Constructs a new back-off handler with the given back-off.</summary>
 /// <param name="backOff">The back-off policy.</param>
 public BackOffHandler(IBackOff backOff)
     : this(new Initializer(backOff))
 {
 }
Пример #10
0
 /// <summary>Constructs a new back-off handler with the given back-off.</summary>
 /// <param name="backOff">The back-off policy.</param>
 public BackOffHandler(IBackOff backOff)
     : this(new Initializer(backOff))
 {
 }
Пример #11
0
 /// <summary>
 /// Constructs a new custom back-off handler
 /// </summary>
 /// <param name="service">Client service</param>
 /// <param name="backoff">Back-off strategy (e.g. exponential backoff)</param>
 public BackoffHandler(IClientService service, IBackOff backoff)
 {
     _service     = service;
     _backoff     = backoff;
     _maxTimeSpan = TimeSpan.FromHours(1);
 }
Пример #12
0
 /// <summary>Constructs a new back-off handler by the given back-off.</summary>
 public MockBackOffHandler(IBackOff backOff)
     : base(backOff)
 {
     Waits = new List<TimeSpan>();
 }
Пример #13
0
 /// <summary> Constructs a new back-off handler by the given back-off. </summary>
 public MockBackOffHandler(IBackOff backOff)
     : base(backOff)
 {
     Waits = new List <TimeSpan>();
 }
Пример #14
0
        /// <param name="retryStrategy">If not null, try retry.</param>
        public async Task InsertAllAsync <T>(BigqueryService service, IEnumerable <T> data, IBackOff retryStrategy = null, Func <T, string> insertIdSelector = null, JsonSerializerSettings serializerSettings = null)
        {
            if (insertIdSelector == null)
            {
                insertIdSelector = _ => Guid.NewGuid().ToString();
            }

            var rows = data
                       .Select(x => new Google.Apis.Bigquery.v2.Data.TableDataInsertAllRequest.RowsData
            {
                InsertId = insertIdSelector(x),
                Json     = JsonConvert.DeserializeObject <Dictionary <string, object> >(JsonConvert.SerializeObject(x, serializerSettings))
            })
                       .Where(x => x.Json != null)
                       .ToArray();

            if (!rows.Any())
            {
                return;
            }

            var request = service.Tabledata.InsertAll(new TableDataInsertAllRequest
            {
                Rows = rows
            }, this.project_id, this.dataset_id, this.table_id);

            var retry = 0;
            TableDataInsertAllResponse response = null;
            Exception lastError;

            do
            {
                try
                {
                    lastError = null;
                    response  = await request.ExecuteAsync().ConfigureAwait(false);
                }
                catch (Exception ex)
                {
                    lastError = ex;
                }

                if (retryStrategy == null)
                {
                    break;
                }
                if (response != null && response.InsertErrors == null)
                {
                    break;
                }

                retry++;
                var nextDelay = retryStrategy.GetNextBackOff(retry);
                if (nextDelay == TimeSpan.MinValue)
                {
                    break;
                }

                await Task.Delay(nextDelay).ConfigureAwait(false);
            } while (true);

            if (lastError != null)
            {
                var exception = new InsertAllFailedException("", lastError)
                {
                    RetryCount         = retry,
                    InternalErrorInfos = new InsertAllFailedException.ErrorInfo[0],
                };

                throw exception;
            }

            if (response.InsertErrors != null && response.InsertErrors.Any())
            {
                var errorMessages = response.InsertErrors.Zip(rows, (x, r) =>
                {
                    return(x.Errors.Select(e =>
                    {
                        return new InsertAllFailedException.ErrorInfo
                        {
                            Index = x.Index,
                            DebugInfo = e.DebugInfo,
                            ETag = e.ETag,
                            Location = e.Location,
                            Message = e.Message,
                            Reason = e.Reason,
                            PostRawJSON = JsonConvert.SerializeObject(r.Json, Formatting.None)
                        };
                    }));
                }).SelectMany(xs => xs);

                var exception = new InsertAllFailedException
                {
                    RetryCount         = retry,
                    InternalErrorInfos = errorMessages.ToArray()
                };

                throw exception;
            }
        }
Пример #15
0
        /// <summary>
        /// Create sink
        /// </summary>
        /// <param name="projectId">Project id</param>
        /// <param name="datasetId">Dataset id</param>
        /// <param name="tableId">Table id. Expand through DateTime.Format(). e.g. "accesslogyyyyMMdd" => accesslog20150101 <see cref="https://msdn.microsoft.com/en-us/library/vstudio/zdtaw1bw(v=vs.100).aspx"/></param>
        /// <param name="authMethod">private_key</param>
        /// <param name="serviceAccountEmail">000000000000-xxxxxxxxxxxxxxxxxxxxxx@developer.gserviceaccount.com</param>
        /// <param name="privateKeyFile">/path/to/xxxx-000000000000.p12</param>
        /// <param name="privateKeyPassphrase">notasecret</param>
        /// <param name="autoCreateTable">Create table if it does not exists</param>
        /// <param name="tableSchemaFile">Json file path that bigquery table schema</param>
        /// <param name="insertIdFieldName">The field name of InsertId</param>
        /// <param name="bufferingInterval"></param>
        /// <param name="bufferingCount"></param>
        /// <param name="bufferingFlushAllTimeout"></param>
        /// <param name="maxBufferSize"></param>
        public BigQuerySink(
            string projectId,
            string datasetId,
            string tableId,
            string authMethod                 = null,
            string serviceAccountEmail        = null,
            string privateKeyFile             = null,
            string privateKeyPassphrase       = null,
            bool?autoCreateTable              = null,
            string tableSchemaFile            = null,
            string insertIdFieldName          = null,
            TimeSpan?bufferingInterval        = null,
            int?bufferingCount                = null,
            TimeSpan?bufferingFlushAllTimeout = null,
            int?maxBufferSize                 = null)
        {
            if (authMethod == null)
            {
                authMethod = "private_key";
            }
            if (authMethod != "private_key")
            {
                throw new NotSupportedException("authMethod must be 'private_key'");
            }
            if (String.IsNullOrEmpty(serviceAccountEmail))
            {
                throw new ArgumentException("serviceAccountEmail");
            }
            if (String.IsNullOrEmpty(privateKeyFile))
            {
                throw new ArgumentException("privateKeyFile");
            }
            if (privateKeyPassphrase == null)
            {
                privateKeyPassphrase = "notasecret";
            }
            if (String.IsNullOrEmpty(projectId))
            {
                throw new ArgumentException("projectId");
            }
            if (String.IsNullOrEmpty(datasetId))
            {
                throw new ArgumentException("datasetId");
            }
            if (String.IsNullOrEmpty(tableId))
            {
                throw new ArgumentException("tableId");
            }
            if (bufferingInterval == null)
            {
                bufferingInterval = Constants.DefaultBufferingInterval;
            }
            if (bufferingCount == null)
            {
                bufferingCount = Constants.DefaultBufferingCount;
            }
            if (bufferingFlushAllTimeout == null)
            {
                bufferingFlushAllTimeout = Constants.DefaultBufferingFlushAllTimeout;
            }
            if (maxBufferSize == null)
            {
                maxBufferSize = Constants.DefaultMaxBufferSize;
            }
            this.ProjectId         = projectId;
            this.DatasetId         = datasetId;
            this.TableId           = tableId;
            this.AutoCreateTable   = autoCreateTable ?? false;
            this.TableSchemaFile   = tableSchemaFile;
            this.InsertIdFieldName = insertIdFieldName;

            // Load table schema file
            if (TableSchemaFile != null)
            {
                TableSchema = LoadTableSchema(TableSchemaFile);
            }
            if (TableSchema == null)
            {
                throw new Exception("table schema not set");
            }
            // Expand table id 1st time within force mode
            ExpandTableIdIfNecessary(force: true);
            // configure finished
            BigQuerySinkEventSource.Log.SinkStarted("TableId: " + TableIdExpanded);

            // Setup bigquery client
            var certificate = new X509Certificate2(
                privateKeyFile,
                privateKeyPassphrase,
                X509KeyStorageFlags.MachineKeySet | X509KeyStorageFlags.Exportable);

            ServiceAccountCredential credential = new ServiceAccountCredential(
                new ServiceAccountCredential.Initializer(serviceAccountEmail)
            {
                Scopes = new[] { BigqueryService.Scope.Bigquery,
                                 BigqueryService.Scope.BigqueryInsertdata,
                                 BigqueryService.Scope.CloudPlatform,
                                 BigqueryService.Scope.DevstorageFullControl }
            }.FromCertificate(certificate));

            _BQSvc = new BigqueryService(new BaseClientService.Initializer
            {
                HttpClientInitializer = credential,
            });
            _BackOff = new ExponentialBackOff();
            _bufferingFlushAllTimeout = bufferingFlushAllTimeout.Value;
            _bufferedPublisher        = BufferedEventPublisher <EventEntry> .CreateAndStart(
                "BigQuery",
                PublishEventsAsync,
                bufferingInterval.Value,
                bufferingCount.Value,
                maxBufferSize.Value,
                _cancellationTokenSource.Token);
        }