public BackOffController(ICounter counter, IBackOff dac, ITime time, IOTP otp)
{
this.counter = counter;
this.dac = dac;
this.time = time;
this.otp = otp;
}
/// <summary>Constructs a new initializer by the given back-off.</summary>
public Initializer(IBackOff backOff)
{
BackOff = backOff;
HandleExceptionFunc = DefaultHandleExceptionFunc;
HandleUnsuccessfulResponseFunc = DefaultHandleUnsuccessfulResponseFunc;
MaxTimeSpan = TimeSpan.FromSeconds(16);
}
/// <summary>Constructs a new initializer by the given back-off.</summary>
public Initializer(IBackOff backOff)
{
BackOff = backOff;
HandleExceptionFunc = DefaultHandleExceptionFunc;
HandleUnsuccessfulResponseFunc = DefaultHandleUnsuccessfulResponseFunc;
MaxTimeSpan = TimeSpan.FromSeconds(16);
}
public Synchronizer(ISplitFetcher splitFetcher,
ISelfRefreshingSegmentFetcher segmentFetcher,
IImpressionsLog impressionsLog,
IEventsLog eventsLog,
IImpressionsCountSender impressionsCountSender,
IWrapperAdapter wrapperAdapter,
IStatusManager statusManager,
ITelemetrySyncTask telemetrySyncTask,
ITasksManager tasksManager,
ISplitCache splitCache,
IBackOff backOff,
int onDemandFetchMaxRetries,
int onDemandFetchRetryDelayMs,
ISegmentCache segmentCache,
ISplitLogger log = null)
{
_splitFetcher = splitFetcher;
_segmentFetcher = segmentFetcher;
_impressionsLog = impressionsLog;
_eventsLog = eventsLog;
_impressionsCountSender = impressionsCountSender;
_wrapperAdapter = wrapperAdapter;
_statusManager = statusManager;
_telemetrySyncTask = telemetrySyncTask;
_tasksManager = tasksManager;
_splitCache = splitCache;
_backOffSplits = backOff;
_backOffSegments = backOff;
_onDemandFetchMaxRetries = onDemandFetchMaxRetries;
_onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs;
_segmentCache = segmentCache;
_log = log ?? WrapperAdapter.GetLogger(typeof(Synchronizer));
_defaultFetchOptions = new FetchOptions();
}
public PushManager(int authRetryBackOffBase,
ISSEHandler sseHandler,
IAuthApiClient authApiClient,
IWrapperAdapter wrapperAdapter = null,
ISplitLogger log = null,
IBackOff backOff = null)
{
_sseHandler = sseHandler;
_authApiClient = authApiClient;
_log = log ?? WrapperAdapter.GetLogger(typeof(PushManager));
_wrapperAdapter = wrapperAdapter ?? new WrapperAdapter();
_backOff = backOff ?? new BackOff(authRetryBackOffBase, attempt: 1);
}
public EventSourceClient(int backOffBase,
ISplitLogger log = null,
INotificationParser notificationParser = null,
IBackOff backOff = null,
IWrapperAdapter wrapperAdapter = null)
{
_log = log ?? WrapperAdapter.GetLogger(typeof(EventSourceClient));
_notificationParser = notificationParser ?? new NotificationParser();
_backOff = backOff ?? new BackOff(backOffBase);
_wrapperAdapter = wrapperAdapter ?? new WrapperAdapter();
UpdateFinishedConnection(finished: true);
}
public PushManager(ISSEHandler sseHandler,
IAuthApiClient authApiClient,
IWrapperAdapter wrapperAdapter,
ITelemetryRuntimeProducer telemetryRuntimeProducer,
IBackOff backOff,
ISplitLogger log = null)
{
_sseHandler = sseHandler;
_authApiClient = authApiClient;
_log = log ?? WrapperAdapter.GetLogger(typeof(PushManager));
_wrapperAdapter = wrapperAdapter;
_backOff = backOff;
_telemetryRuntimeProducer = telemetryRuntimeProducer;
}
private void Initialize(BigQueryOutputConfiguration bqOutputConfiguration)
{
Debug.Assert(bqOutputConfiguration != null);
Debug.Assert(this.healthReporter != null);
if (String.IsNullOrEmpty(bqOutputConfiguration.ProjectId))
{
throw new ArgumentException("ProjectId");
}
if (String.IsNullOrEmpty(bqOutputConfiguration.DatasetId))
{
throw new ArgumentException("DatasetId");
}
if (String.IsNullOrEmpty(bqOutputConfiguration.TableId))
{
throw new ArgumentException("TableId");
}
this.Config = bqOutputConfiguration;
// Load table schema file
if (Config.TableSchemaFile != null)
{
TableSchema = LoadTableSchema(Config.TableSchemaFile);
}
if (TableSchema == null)
{
throw new Exception("table schema not set");
}
// Expand table id 1st time within force mode
ExpandTableIdIfNecessary(force: true);
// configure finished
healthReporter.ReportHealthy("TableId: " + TableIdExpanded);
var scopes = new[]
{
BigqueryService.Scope.Bigquery,
BigqueryService.Scope.BigqueryInsertdata,
BigqueryService.Scope.CloudPlatform,
BigqueryService.Scope.DevstorageFullControl
};
var credential = GoogleCredential.GetApplicationDefault()
.CreateScoped(scopes);
_BQSvc = new BigqueryService(new BaseClientService.Initializer
{
HttpClientInitializer = credential,
});
_BackOff = new ExponentialBackOff();
}
/// <summary>Constructs a new back-off handler with the given back-off.</summary>
/// <param name="backOff">The back-off policy.</param>
public BackOffHandler(IBackOff backOff)
: this(new Initializer(backOff))
{
}
/// <summary>Constructs a new back-off handler with the given back-off.</summary>
/// <param name="backOff">The back-off policy.</param>
public BackOffHandler(IBackOff backOff)
: this(new Initializer(backOff))
{
}
/// <summary>
/// Constructs a new custom back-off handler
/// </summary>
/// <param name="service">Client service</param>
/// <param name="backoff">Back-off strategy (e.g. exponential backoff)</param>
public BackoffHandler(IClientService service, IBackOff backoff)
{
_service = service;
_backoff = backoff;
_maxTimeSpan = TimeSpan.FromHours(1);
}
/// <summary>Constructs a new back-off handler by the given back-off.</summary>
public MockBackOffHandler(IBackOff backOff)
: base(backOff)
{
Waits = new List<TimeSpan>();
}
/// <summary> Constructs a new back-off handler by the given back-off. </summary>
public MockBackOffHandler(IBackOff backOff)
: base(backOff)
{
Waits = new List <TimeSpan>();
}
/// <param name="retryStrategy">If not null, try retry.</param>
public async Task InsertAllAsync <T>(BigqueryService service, IEnumerable <T> data, IBackOff retryStrategy = null, Func <T, string> insertIdSelector = null, JsonSerializerSettings serializerSettings = null)
{
if (insertIdSelector == null)
{
insertIdSelector = _ => Guid.NewGuid().ToString();
}
var rows = data
.Select(x => new Google.Apis.Bigquery.v2.Data.TableDataInsertAllRequest.RowsData
{
InsertId = insertIdSelector(x),
Json = JsonConvert.DeserializeObject <Dictionary <string, object> >(JsonConvert.SerializeObject(x, serializerSettings))
})
.Where(x => x.Json != null)
.ToArray();
if (!rows.Any())
{
return;
}
var request = service.Tabledata.InsertAll(new TableDataInsertAllRequest
{
Rows = rows
}, this.project_id, this.dataset_id, this.table_id);
var retry = 0;
TableDataInsertAllResponse response = null;
Exception lastError;
do
{
try
{
lastError = null;
response = await request.ExecuteAsync().ConfigureAwait(false);
}
catch (Exception ex)
{
lastError = ex;
}
if (retryStrategy == null)
{
break;
}
if (response != null && response.InsertErrors == null)
{
break;
}
retry++;
var nextDelay = retryStrategy.GetNextBackOff(retry);
if (nextDelay == TimeSpan.MinValue)
{
break;
}
await Task.Delay(nextDelay).ConfigureAwait(false);
} while (true);
if (lastError != null)
{
var exception = new InsertAllFailedException("", lastError)
{
RetryCount = retry,
InternalErrorInfos = new InsertAllFailedException.ErrorInfo[0],
};
throw exception;
}
if (response.InsertErrors != null && response.InsertErrors.Any())
{
var errorMessages = response.InsertErrors.Zip(rows, (x, r) =>
{
return(x.Errors.Select(e =>
{
return new InsertAllFailedException.ErrorInfo
{
Index = x.Index,
DebugInfo = e.DebugInfo,
ETag = e.ETag,
Location = e.Location,
Message = e.Message,
Reason = e.Reason,
PostRawJSON = JsonConvert.SerializeObject(r.Json, Formatting.None)
};
}));
}).SelectMany(xs => xs);
var exception = new InsertAllFailedException
{
RetryCount = retry,
InternalErrorInfos = errorMessages.ToArray()
};
throw exception;
}
}
/// <summary>
/// Create sink
/// </summary>
/// <param name="projectId">Project id</param>
/// <param name="datasetId">Dataset id</param>
/// <param name="tableId">Table id. Expand through DateTime.Format(). e.g. "accesslogyyyyMMdd" => accesslog20150101 <see cref="https://msdn.microsoft.com/en-us/library/vstudio/zdtaw1bw(v=vs.100).aspx"/></param>
/// <param name="authMethod">private_key</param>
/// <param name="serviceAccountEmail">000000000000-xxxxxxxxxxxxxxxxxxxxxx@developer.gserviceaccount.com</param>
/// <param name="privateKeyFile">/path/to/xxxx-000000000000.p12</param>
/// <param name="privateKeyPassphrase">notasecret</param>
/// <param name="autoCreateTable">Create table if it does not exists</param>
/// <param name="tableSchemaFile">Json file path that bigquery table schema</param>
/// <param name="insertIdFieldName">The field name of InsertId</param>
/// <param name="bufferingInterval"></param>
/// <param name="bufferingCount"></param>
/// <param name="bufferingFlushAllTimeout"></param>
/// <param name="maxBufferSize"></param>
public BigQuerySink(
string projectId,
string datasetId,
string tableId,
string authMethod = null,
string serviceAccountEmail = null,
string privateKeyFile = null,
string privateKeyPassphrase = null,
bool?autoCreateTable = null,
string tableSchemaFile = null,
string insertIdFieldName = null,
TimeSpan?bufferingInterval = null,
int?bufferingCount = null,
TimeSpan?bufferingFlushAllTimeout = null,
int?maxBufferSize = null)
{
if (authMethod == null)
{
authMethod = "private_key";
}
if (authMethod != "private_key")
{
throw new NotSupportedException("authMethod must be 'private_key'");
}
if (String.IsNullOrEmpty(serviceAccountEmail))
{
throw new ArgumentException("serviceAccountEmail");
}
if (String.IsNullOrEmpty(privateKeyFile))
{
throw new ArgumentException("privateKeyFile");
}
if (privateKeyPassphrase == null)
{
privateKeyPassphrase = "notasecret";
}
if (String.IsNullOrEmpty(projectId))
{
throw new ArgumentException("projectId");
}
if (String.IsNullOrEmpty(datasetId))
{
throw new ArgumentException("datasetId");
}
if (String.IsNullOrEmpty(tableId))
{
throw new ArgumentException("tableId");
}
if (bufferingInterval == null)
{
bufferingInterval = Constants.DefaultBufferingInterval;
}
if (bufferingCount == null)
{
bufferingCount = Constants.DefaultBufferingCount;
}
if (bufferingFlushAllTimeout == null)
{
bufferingFlushAllTimeout = Constants.DefaultBufferingFlushAllTimeout;
}
if (maxBufferSize == null)
{
maxBufferSize = Constants.DefaultMaxBufferSize;
}
this.ProjectId = projectId;
this.DatasetId = datasetId;
this.TableId = tableId;
this.AutoCreateTable = autoCreateTable ?? false;
this.TableSchemaFile = tableSchemaFile;
this.InsertIdFieldName = insertIdFieldName;
// Load table schema file
if (TableSchemaFile != null)
{
TableSchema = LoadTableSchema(TableSchemaFile);
}
if (TableSchema == null)
{
throw new Exception("table schema not set");
}
// Expand table id 1st time within force mode
ExpandTableIdIfNecessary(force: true);
// configure finished
BigQuerySinkEventSource.Log.SinkStarted("TableId: " + TableIdExpanded);
// Setup bigquery client
var certificate = new X509Certificate2(
privateKeyFile,
privateKeyPassphrase,
X509KeyStorageFlags.MachineKeySet | X509KeyStorageFlags.Exportable);
ServiceAccountCredential credential = new ServiceAccountCredential(
new ServiceAccountCredential.Initializer(serviceAccountEmail)
{
Scopes = new[] { BigqueryService.Scope.Bigquery,
BigqueryService.Scope.BigqueryInsertdata,
BigqueryService.Scope.CloudPlatform,
BigqueryService.Scope.DevstorageFullControl }
}.FromCertificate(certificate));
_BQSvc = new BigqueryService(new BaseClientService.Initializer
{
HttpClientInitializer = credential,
});
_BackOff = new ExponentialBackOff();
_bufferingFlushAllTimeout = bufferingFlushAllTimeout.Value;
_bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(
"BigQuery",
PublishEventsAsync,
bufferingInterval.Value,
bufferingCount.Value,
maxBufferSize.Value,
_cancellationTokenSource.Token);
}