public Task <IDataSinkAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { ValidateBaseConfiguration(configuration); return(CreateAsync( context, GetDataItemTransformation(configuration), configuration, cancellation)); }
public Task <IDataSinkAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { ValidateBaseConfiguration(configuration); var collectionNames = ResolveCollectionNames(configuration.Collection); if (!collectionNames.Any()) { throw Errors.CollectionNameMissing(); } return(CreateAsync( CreateClient(configuration, context, collectionNames.Length > 1), GetDataItemTransformation(configuration), configuration, collectionNames, cancellation)); }
protected override async Task <IDataSinkAdapter> CreateAsync(IDataTransferContext context, IDataItemTransformation transformation, IDocumentDbParallelSinkAdapterConfiguration configuration, CancellationToken cancellation) { if (String.IsNullOrEmpty(configuration.Collection)) { throw Errors.CollectionNameMissing(); } var sink = new DocumentDbParallelSinkAdapter( CreateClient(configuration, context, false), transformation, GetInstanceConfiguration(configuration)); await sink.InitializeAsync(); return(sink); }
protected override async Task <IDataSinkAdapter> CreateAsync(IDataTransferContext context, IDataItemTransformation transformation, IDocumentDbBulkSinkAdapterConfiguration configuration, CancellationToken cancellation) { var collectionNames = ResolveCollectionNames(configuration.Collection); if (!collectionNames.Any()) { throw Errors.CollectionNameMissing(); } var sink = new DocumentDbBulkSinkAdapterDispatcher( CreateClient(configuration, context, collectionNames.Count() > 1, null), transformation, GetInstanceConfiguration(configuration, collectionNames)); await sink.InitializeAsync(cancellation); return(sink); }
public Task <TDataAdapter> CreateAsync(object configuration, IDataTransferContext context) { if (configuration != null && !ConfigurationType.IsAssignableFrom(configuration.GetType())) { throw Errors.InvalidDataAdapterConfigrationType(ConfigurationType, configuration.GetType()); } try { return((Task <TDataAdapter>)createMethod.Invoke(factory, new[] { configuration, context })); } catch (TargetInvocationException invocationException) { if (invocationException.InnerException != null) { throw invocationException.InnerException; } throw; } }
/// <summary> /// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration. /// </summary> /// <param name="configuration">Data source adapter configuration.</param> /// <param name="context">Data transfer operation context.</param> /// <param name="cancellation">Cancellation token.</param> /// <returns>Task that represents asynchronous create operation.</returns> public async Task <IDataSourceAdapter> CreateAsync(IMongoDbSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { Guard.NotNull("configuration", configuration); if (String.IsNullOrEmpty(configuration.ConnectionString)) { throw Errors.ConnectionStringMissing(); } if (String.IsNullOrEmpty(configuration.Collection)) { throw Errors.CollectionNameMissing(); } var adapter = new MongoDbSourceAdapter(GetInstanceConfiguration(configuration)); await adapter.Initialize(cancellation); return(adapter); }
private static DocumentClient CreateRawClient(IDocumentDbConnectionSettings connectionSettings, DocumentDbConnectionMode?connectionMode, IDataTransferContext context) { Guard.NotNull("connectionSettings", connectionSettings); return(new DocumentClient( new Uri(connectionSettings.AccountEndpoint), connectionSettings.AccountKey, CreateConnectionPolicy(connectionMode, context) )); }
/// <summary> /// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration. /// </summary> /// <param name="configuration">Data source adapter configuration.</param> /// <param name="context">Data transfer operation context.</param> /// <param name="cancellation">Cancellation token.</param> /// <returns>Task that represents asynchronous create operation.</returns> public Task <IDataSourceAdapter> CreateAsync(IAzureTableSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { return(Task.Factory.StartNew(() => Create(configuration), cancellation)); }
private IDataSourceAdapter Create(IDocumentDbSourceAdapterConfiguration configuration, IDataTransferContext context) { Guard.NotNull("configuration", configuration); ValidateBaseConfiguration(configuration); var source = new DocumentDbSourceAdapter( CreateClient(configuration, context), GetDataItemTransformation(configuration), GetInstanceConfiguration(configuration)); source.Initialize(); return(source); }
public async Task <IDataSourceAdapter> CreateAsync(IDocumentDbSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { Guard.NotNull("configuration", configuration); ValidateBaseConfiguration(configuration); var source = new DocumentDbSourceAdapter( CreateClient(configuration, context, false), GetDataItemTransformation(configuration), GetInstanceConfiguration(configuration)); await source.InitializeAsync(cancellation); return(source); }
public Task <IDataSourceAdapter> CreateAsync(IDynamoDbSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { return(Task.Factory.StartNew <IDataSourceAdapter>(Create, configuration, cancellation)); }
private static ConnectionPolicy CreateConnectionPolicy(DocumentDbConnectionMode?connectionMode, IDataTransferContext context, bool isShardedImport) { var entryAssembly = Assembly.GetEntryAssembly(); return(DocumentDbClientHelper.ApplyConnectionMode(new ConnectionPolicy { UserAgentSuffix = String.Format(CultureInfo.InvariantCulture, Resources.CustomUserAgentSuffixFormat, entryAssembly == null ? Resources.UnknownEntryAssembly : entryAssembly.GetName().Name, Assembly.GetExecutingAssembly().GetName().Version, context.SourceName, context.SinkName, isShardedImport ? Resources.ShardedImportDesignator : String.Empty) }, connectionMode)); }
protected abstract Task <IDataSinkAdapter> CreateAsync(IDataTransferContext context, IDataItemTransformation transformation, TConfiguration configuration, CancellationToken cancellation);
public async Task <IDataSourceAdapter> CreateAsync(IHBaseSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { Guard.NotNull("configuration", configuration); ValidateBaseConfiguration(configuration); if (String.IsNullOrEmpty(configuration.Table)) { throw Errors.TableNameMissing(); } var adapter = new HBaseSourceAdapter(StargateClientFactory.Create(configuration.ConnectionString), GetInstanceConfiguration(configuration)); await adapter.Initialize(cancellation); return(adapter); }
/// <summary> /// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration. /// </summary> /// <param name="configuration">Data source adapter configuration.</param> /// <param name="context">Data transfer operation context.</param> /// <returns>Task that represents asynchronous create operation.</returns> public Task <IDataSourceAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context) { return(Factory.CreateAsync(configuration, context)); }
/// <summary> /// Creates a new instance of <see cref="IDataSinkAdapter" /> with the provided configuration. /// </summary> /// <param name="configuration">Data sink adapter configuration.</param> /// <param name="context">Data transfer operation context.</param> /// <param name="cancellation">Cancellation token.</param> /// <returns>Task that represents asynchronous create operation.</returns> public Task <IDataSinkAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { return(Factory.CreateAsync(configuration, context, cancellation)); }
/// <summary> /// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration. /// </summary> /// <param name="configuration">Data source adapter configuration.</param> /// <param name="context">Data transfer operation context.</param> /// <returns>Task that represents asynchronous create operation.</returns> public Task <IDataSourceAdapter> CreateAsync(IJsonFileSourceAdapterConfiguration configuration, IDataTransferContext context) { return(Task.Factory.StartNew(() => Create(configuration))); }
/// <summary> /// Creates a new instance of <see cref="IDataSinkAdapter" /> with the provided configuration. /// </summary> /// <param name="configuration">Data sink adapter configuration.</param> /// <param name="context">Data transfer operation context.</param> /// <param name="cancellation">Cancellation token.</param> /// <returns>Task that represents asynchronous create operation.</returns> public async Task <IDataSinkAdapter> CreateAsync(IJsonFileSinkAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { Guard.NotNull("configuration", configuration); return(new JsonFileSinkAdapter( new StreamWriter( await SinkStreamProvidersFactory.Create(configuration.File, configuration.Compress, configuration.Overwrite) .CreateStream(cancellation)), JsonSerializersFactory.Create(configuration.Prettify))); }
public Task <IDataSourceAdapter> CreateAsync(IDocumentDbSourceAdapterConfiguration configuration, IDataTransferContext context) { return(Task.Factory.StartNew <IDataSourceAdapter>(() => Create(configuration, context))); }
/// <summary> /// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration. /// </summary> /// <param name="configuration">Data source adapter configuration.</param> /// <param name="context">Data transfer operation context.</param> /// <returns>Task that represents asynchronous create operation.</returns> public async Task <IDataSourceAdapter> CreateAsync(ISqlDataSourceAdapterConfiguration configuration, IDataTransferContext context) { Guard.NotNull("configuration", configuration); if (String.IsNullOrEmpty(configuration.ConnectionString)) { throw Errors.ConnectionStringMissing(); } var adapter = new SqlQueryDataSourceAdapter(GetInstanceConfiguration(configuration)); await adapter.InitializeAsync(); return(adapter); }
/// <summary> /// Creates a new instance of <see cref="IDataSinkAdapter" /> with the provided configuration. /// </summary> /// <param name="configuration">Data sink adapter configuration.</param> /// <param name="context">Data transfer operation context.</param> /// <param name="cancellation">Cancellation token.</param> /// <returns>Task that represents asynchronous create operation.</returns> public async Task <IDataSinkAdapter> CreateAsync(ITableAPIBulkSinkAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { Guard.NotNull("configuration", configuration); if (String.IsNullOrEmpty(configuration.ConnectionString)) { throw Errors.ConnectionStringMissing(); } if (String.IsNullOrEmpty(configuration.TableName)) { throw Errors.TableNameMissing(); } long maxInputBufferSizeInBytes = 10 * 1024 * 1024; if (configuration.MaxInputBufferSize.HasValue) { maxInputBufferSizeInBytes = configuration.MaxInputBufferSize.Value; } int throughput = 400; if (configuration.Throughput.HasValue) { throughput = configuration.Throughput.Value; } const int maxBatchSize = 2 * 1024 * 1024; int batchSize = maxBatchSize; if (configuration.MaxBatchSize.HasValue) { if (configuration.MaxBatchSize.Value > maxBatchSize) { throw Errors.BatchSizeInvalid(maxBatchSize); } batchSize = configuration.MaxBatchSize.Value; } if (configuration.ConnectionString.EndsWith("/")) { configuration.ConnectionString.TrimEnd('/'); } var sink = new TableAPIBulkSinkAdapter(configuration.ConnectionString, configuration.TableName, configuration.Overwrite, maxInputBufferSizeInBytes, throughput, batchSize); await sink.InitializeAsync(cancellation); return(sink); }
protected static DocumentDbClient CreateClient(IDocumentDbAdapterConfiguration configuration, IDataTransferContext context, bool isShardedImport, int?maxConnectionLimit) { Guard.NotNull("configuration", configuration); var connectionSettings = ParseConnectionString(configuration.ConnectionString); return(new DocumentDbClient( CreateRawClient(connectionSettings, configuration.ConnectionMode, context, isShardedImport, maxConnectionLimit, configuration.Retries, configuration.RetryInterval), connectionSettings.Database )); }
protected static DocumentDbClient CreateClient(IDocumentDbAdapterConfiguration configuration, IDataTransferContext context) { Guard.NotNull("configuration", configuration); var connectionSettings = ParseConnectionString(configuration.ConnectionString); return(new DocumentDbClient( CreateRawClient(connectionSettings, configuration.ConnectionMode, context) .AsReliable(new FixedInterval( null, GetValueOrDefault(configuration.Retries, Defaults.Current.NumberOfRetries, Errors.InvalidNumberOfRetries), GetValueOrDefault(configuration.RetryInterval, Defaults.Current.RetryInterval, Errors.InvalidRetryInterval), false)), connectionSettings.Database )); }
private static DocumentClient CreateRawClient(IDocumentDbConnectionSettings connectionSettings, DocumentDbConnectionMode?connectionMode, IDataTransferContext context, bool isShardedImport, int?maxConnectionLimit, int?retries, TimeSpan?retryInterval) { Guard.NotNull("connectionSettings", connectionSettings); return(new DocumentClient( new Uri(connectionSettings.AccountEndpoint), connectionSettings.AccountKey, CreateConnectionPolicy(connectionMode, context, isShardedImport, maxConnectionLimit, retries, retryInterval) )); }
private static ConnectionPolicy CreateConnectionPolicy(DocumentDbConnectionMode?connectionMode, IDataTransferContext context) { return(DocumentDbClientHelper.ApplyConnectionMode(new ConnectionPolicy { UserAgentSuffix = String.Format(CultureInfo.InvariantCulture, Resources.CustomUserAgentSuffixFormat, Assembly.GetExecutingAssembly().GetName().Version, context.SourceName, context.SinkName) }, connectionMode)); }
private static ConnectionPolicy CreateConnectionPolicy(DocumentDbConnectionMode?connectionMode, IDataTransferContext context, bool isShardedImport, int?maxConnectionLimit, int?retries, TimeSpan?retryInterval) { var entryAssembly = Assembly.GetEntryAssembly(); var connectionPolicy = new ConnectionPolicy { UserAgentSuffix = String.Format(CultureInfo.InvariantCulture, Resources.CustomUserAgentSuffixFormat, entryAssembly == null ? Resources.UnknownEntryAssembly : entryAssembly.GetName().Name, Assembly.GetExecutingAssembly().GetName().Version, context.SourceName, context.SinkName, isShardedImport ? Resources.ShardedImportDesignator : String.Empty) }; RetryOptions retryOptions = new RetryOptions(); if (retries.HasValue) { retryOptions.MaxRetryAttemptsOnThrottledRequests = retries.Value; } if (retryInterval.HasValue) { retryOptions.MaxRetryWaitTimeInSeconds = retryInterval.Value.Seconds; } connectionPolicy.RetryOptions = retryOptions; if (maxConnectionLimit.HasValue) { connectionPolicy.MaxConnectionLimit = maxConnectionLimit.Value; } return(DocumentDbClientHelper.ApplyConnectionMode(connectionPolicy, connectionMode)); }
public Task <IDataSinkAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context) { ValidateBaseConfiguration(configuration); return(CreateAsync(CreateClient(configuration, context), GetDataItemTransformation(configuration), configuration)); }
private static DocumentClient CreateRawClient(IDocumentDbConnectionSettings connectionSettings, DocumentDbConnectionMode?connectionMode, IDataTransferContext context, bool isShardedImport, int?maxConnectionLimit, int?retries, TimeSpan?retryInterval, bool?ignoreSSLCertErrors = false) { Guard.NotNull("connectionSettings", connectionSettings); return(new DocumentClient( new Uri(connectionSettings.AccountEndpoint), connectionSettings.AccountKey, ignoreSSLCertErrors.GetValueOrDefault() ? DocumentDbClientHelper.GetSSLCertHandler() : null, !ignoreSSLCertErrors.GetValueOrDefault() ? CreateConnectionPolicy(connectionMode, context, isShardedImport, maxConnectionLimit, retries, retryInterval) : null )); }