public Task <IDataSinkAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
ValidateBaseConfiguration(configuration);
return(CreateAsync(
context, GetDataItemTransformation(configuration),
configuration, cancellation));
}
public Task <IDataSinkAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
ValidateBaseConfiguration(configuration);
var collectionNames = ResolveCollectionNames(configuration.Collection);
if (!collectionNames.Any())
{
throw Errors.CollectionNameMissing();
}
return(CreateAsync(
CreateClient(configuration, context, collectionNames.Length > 1),
GetDataItemTransformation(configuration),
configuration, collectionNames, cancellation));
}
protected override async Task <IDataSinkAdapter> CreateAsync(IDataTransferContext context, IDataItemTransformation transformation,
IDocumentDbParallelSinkAdapterConfiguration configuration, CancellationToken cancellation)
{
if (String.IsNullOrEmpty(configuration.Collection))
{
throw Errors.CollectionNameMissing();
}
var sink = new DocumentDbParallelSinkAdapter(
CreateClient(configuration, context, false), transformation,
GetInstanceConfiguration(configuration));
await sink.InitializeAsync();
return(sink);
}
protected override async Task <IDataSinkAdapter> CreateAsync(IDataTransferContext context, IDataItemTransformation transformation,
IDocumentDbBulkSinkAdapterConfiguration configuration, CancellationToken cancellation)
{
var collectionNames = ResolveCollectionNames(configuration.Collection);
if (!collectionNames.Any())
{
throw Errors.CollectionNameMissing();
}
var sink = new DocumentDbBulkSinkAdapterDispatcher(
CreateClient(configuration, context, collectionNames.Count() > 1, null),
transformation, GetInstanceConfiguration(configuration, collectionNames));
await sink.InitializeAsync(cancellation);
return(sink);
}
public Task <TDataAdapter> CreateAsync(object configuration, IDataTransferContext context)
{
if (configuration != null && !ConfigurationType.IsAssignableFrom(configuration.GetType()))
{
throw Errors.InvalidDataAdapterConfigrationType(ConfigurationType, configuration.GetType());
}
try
{
return((Task <TDataAdapter>)createMethod.Invoke(factory, new[] { configuration, context }));
}
catch (TargetInvocationException invocationException)
{
if (invocationException.InnerException != null)
{
throw invocationException.InnerException;
}
throw;
}
}
/// <summary>
/// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration.
/// </summary>
/// <param name="configuration">Data source adapter configuration.</param>
/// <param name="context">Data transfer operation context.</param>
/// <param name="cancellation">Cancellation token.</param>
/// <returns>Task that represents asynchronous create operation.</returns>
public async Task <IDataSourceAdapter> CreateAsync(IMongoDbSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
Guard.NotNull("configuration", configuration);
if (String.IsNullOrEmpty(configuration.ConnectionString))
{
throw Errors.ConnectionStringMissing();
}
if (String.IsNullOrEmpty(configuration.Collection))
{
throw Errors.CollectionNameMissing();
}
var adapter = new MongoDbSourceAdapter(GetInstanceConfiguration(configuration));
await adapter.Initialize(cancellation);
return(adapter);
}
private static DocumentClient CreateRawClient(IDocumentDbConnectionSettings connectionSettings, DocumentDbConnectionMode?connectionMode, IDataTransferContext context)
{
Guard.NotNull("connectionSettings", connectionSettings);
return(new DocumentClient(
new Uri(connectionSettings.AccountEndpoint),
connectionSettings.AccountKey,
CreateConnectionPolicy(connectionMode, context)
));
}
/// <summary>
/// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration.
/// </summary>
/// <param name="configuration">Data source adapter configuration.</param>
/// <param name="context">Data transfer operation context.</param>
/// <param name="cancellation">Cancellation token.</param>
/// <returns>Task that represents asynchronous create operation.</returns>
public Task <IDataSourceAdapter> CreateAsync(IAzureTableSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
return(Task.Factory.StartNew(() => Create(configuration), cancellation));
}
private IDataSourceAdapter Create(IDocumentDbSourceAdapterConfiguration configuration, IDataTransferContext context)
{
Guard.NotNull("configuration", configuration);
ValidateBaseConfiguration(configuration);
var source = new DocumentDbSourceAdapter(
CreateClient(configuration, context),
GetDataItemTransformation(configuration),
GetInstanceConfiguration(configuration));
source.Initialize();
return(source);
}
public async Task <IDataSourceAdapter> CreateAsync(IDocumentDbSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
Guard.NotNull("configuration", configuration);
ValidateBaseConfiguration(configuration);
var source = new DocumentDbSourceAdapter(
CreateClient(configuration, context, false),
GetDataItemTransformation(configuration),
GetInstanceConfiguration(configuration));
await source.InitializeAsync(cancellation);
return(source);
}
public Task <IDataSourceAdapter> CreateAsync(IDynamoDbSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
return(Task.Factory.StartNew <IDataSourceAdapter>(Create, configuration, cancellation));
}
private static ConnectionPolicy CreateConnectionPolicy(DocumentDbConnectionMode?connectionMode, IDataTransferContext context, bool isShardedImport)
{
var entryAssembly = Assembly.GetEntryAssembly();
return(DocumentDbClientHelper.ApplyConnectionMode(new ConnectionPolicy
{
UserAgentSuffix = String.Format(CultureInfo.InvariantCulture, Resources.CustomUserAgentSuffixFormat,
entryAssembly == null ? Resources.UnknownEntryAssembly : entryAssembly.GetName().Name,
Assembly.GetExecutingAssembly().GetName().Version,
context.SourceName, context.SinkName,
isShardedImport ? Resources.ShardedImportDesignator : String.Empty)
}, connectionMode));
}
protected abstract Task <IDataSinkAdapter> CreateAsync(IDataTransferContext context, IDataItemTransformation transformation,
TConfiguration configuration, CancellationToken cancellation);
public async Task <IDataSourceAdapter> CreateAsync(IHBaseSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
Guard.NotNull("configuration", configuration);
ValidateBaseConfiguration(configuration);
if (String.IsNullOrEmpty(configuration.Table))
{
throw Errors.TableNameMissing();
}
var adapter = new HBaseSourceAdapter(StargateClientFactory.Create(configuration.ConnectionString), GetInstanceConfiguration(configuration));
await adapter.Initialize(cancellation);
return(adapter);
}
/// <summary>
/// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration.
/// </summary>
/// <param name="configuration">Data source adapter configuration.</param>
/// <param name="context">Data transfer operation context.</param>
/// <returns>Task that represents asynchronous create operation.</returns>
public Task <IDataSourceAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context)
{
return(Factory.CreateAsync(configuration, context));
}
/// <summary>
/// Creates a new instance of <see cref="IDataSinkAdapter" /> with the provided configuration.
/// </summary>
/// <param name="configuration">Data sink adapter configuration.</param>
/// <param name="context">Data transfer operation context.</param>
/// <param name="cancellation">Cancellation token.</param>
/// <returns>Task that represents asynchronous create operation.</returns>
public Task <IDataSinkAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
return(Factory.CreateAsync(configuration, context, cancellation));
}
/// <summary>
/// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration.
/// </summary>
/// <param name="configuration">Data source adapter configuration.</param>
/// <param name="context">Data transfer operation context.</param>
/// <returns>Task that represents asynchronous create operation.</returns>
public Task <IDataSourceAdapter> CreateAsync(IJsonFileSourceAdapterConfiguration configuration, IDataTransferContext context)
{
return(Task.Factory.StartNew(() => Create(configuration)));
}
/// <summary>
/// Creates a new instance of <see cref="IDataSinkAdapter" /> with the provided configuration.
/// </summary>
/// <param name="configuration">Data sink adapter configuration.</param>
/// <param name="context">Data transfer operation context.</param>
/// <param name="cancellation">Cancellation token.</param>
/// <returns>Task that represents asynchronous create operation.</returns>
public async Task <IDataSinkAdapter> CreateAsync(IJsonFileSinkAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
Guard.NotNull("configuration", configuration);
return(new JsonFileSinkAdapter(
new StreamWriter(
await SinkStreamProvidersFactory.Create(configuration.File, configuration.Compress, configuration.Overwrite)
.CreateStream(cancellation)),
JsonSerializersFactory.Create(configuration.Prettify)));
}
public Task <IDataSourceAdapter> CreateAsync(IDocumentDbSourceAdapterConfiguration configuration, IDataTransferContext context)
{
return(Task.Factory.StartNew <IDataSourceAdapter>(() => Create(configuration, context)));
}
/// <summary>
/// Creates a new instance of <see cref="IDataSourceAdapter" /> with the provided configuration.
/// </summary>
/// <param name="configuration">Data source adapter configuration.</param>
/// <param name="context">Data transfer operation context.</param>
/// <returns>Task that represents asynchronous create operation.</returns>
public async Task <IDataSourceAdapter> CreateAsync(ISqlDataSourceAdapterConfiguration configuration, IDataTransferContext context)
{
Guard.NotNull("configuration", configuration);
if (String.IsNullOrEmpty(configuration.ConnectionString))
{
throw Errors.ConnectionStringMissing();
}
var adapter = new SqlQueryDataSourceAdapter(GetInstanceConfiguration(configuration));
await adapter.InitializeAsync();
return(adapter);
}
/// <summary>
/// Creates a new instance of <see cref="IDataSinkAdapter" /> with the provided configuration.
/// </summary>
/// <param name="configuration">Data sink adapter configuration.</param>
/// <param name="context">Data transfer operation context.</param>
/// <param name="cancellation">Cancellation token.</param>
/// <returns>Task that represents asynchronous create operation.</returns>
public async Task <IDataSinkAdapter> CreateAsync(ITableAPIBulkSinkAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation)
{
Guard.NotNull("configuration", configuration);
if (String.IsNullOrEmpty(configuration.ConnectionString))
{
throw Errors.ConnectionStringMissing();
}
if (String.IsNullOrEmpty(configuration.TableName))
{
throw Errors.TableNameMissing();
}
long maxInputBufferSizeInBytes = 10 * 1024 * 1024;
if (configuration.MaxInputBufferSize.HasValue)
{
maxInputBufferSizeInBytes = configuration.MaxInputBufferSize.Value;
}
int throughput = 400;
if (configuration.Throughput.HasValue)
{
throughput = configuration.Throughput.Value;
}
const int maxBatchSize = 2 * 1024 * 1024;
int batchSize = maxBatchSize;
if (configuration.MaxBatchSize.HasValue)
{
if (configuration.MaxBatchSize.Value > maxBatchSize)
{
throw Errors.BatchSizeInvalid(maxBatchSize);
}
batchSize = configuration.MaxBatchSize.Value;
}
if (configuration.ConnectionString.EndsWith("/"))
{
configuration.ConnectionString.TrimEnd('/');
}
var sink = new TableAPIBulkSinkAdapter(configuration.ConnectionString,
configuration.TableName, configuration.Overwrite,
maxInputBufferSizeInBytes, throughput, batchSize);
await sink.InitializeAsync(cancellation);
return(sink);
}
protected static DocumentDbClient CreateClient(IDocumentDbAdapterConfiguration configuration, IDataTransferContext context, bool isShardedImport, int?maxConnectionLimit)
{
Guard.NotNull("configuration", configuration);
var connectionSettings = ParseConnectionString(configuration.ConnectionString);
return(new DocumentDbClient(
CreateRawClient(connectionSettings, configuration.ConnectionMode, context, isShardedImport, maxConnectionLimit, configuration.Retries, configuration.RetryInterval),
connectionSettings.Database
));
}
protected static DocumentDbClient CreateClient(IDocumentDbAdapterConfiguration configuration, IDataTransferContext context)
{
Guard.NotNull("configuration", configuration);
var connectionSettings = ParseConnectionString(configuration.ConnectionString);
return(new DocumentDbClient(
CreateRawClient(connectionSettings, configuration.ConnectionMode, context)
.AsReliable(new FixedInterval(
null,
GetValueOrDefault(configuration.Retries, Defaults.Current.NumberOfRetries, Errors.InvalidNumberOfRetries),
GetValueOrDefault(configuration.RetryInterval, Defaults.Current.RetryInterval, Errors.InvalidRetryInterval),
false)),
connectionSettings.Database
));
}
private static DocumentClient CreateRawClient(IDocumentDbConnectionSettings connectionSettings, DocumentDbConnectionMode?connectionMode, IDataTransferContext context,
bool isShardedImport, int?maxConnectionLimit, int?retries, TimeSpan?retryInterval)
{
Guard.NotNull("connectionSettings", connectionSettings);
return(new DocumentClient(
new Uri(connectionSettings.AccountEndpoint),
connectionSettings.AccountKey,
CreateConnectionPolicy(connectionMode, context, isShardedImport, maxConnectionLimit, retries, retryInterval)
));
}
private static ConnectionPolicy CreateConnectionPolicy(DocumentDbConnectionMode?connectionMode, IDataTransferContext context)
{
return(DocumentDbClientHelper.ApplyConnectionMode(new ConnectionPolicy
{
UserAgentSuffix = String.Format(CultureInfo.InvariantCulture, Resources.CustomUserAgentSuffixFormat,
Assembly.GetExecutingAssembly().GetName().Version, context.SourceName, context.SinkName)
}, connectionMode));
}
private static ConnectionPolicy CreateConnectionPolicy(DocumentDbConnectionMode?connectionMode, IDataTransferContext context, bool isShardedImport, int?maxConnectionLimit, int?retries, TimeSpan?retryInterval)
{
var entryAssembly = Assembly.GetEntryAssembly();
var connectionPolicy =
new ConnectionPolicy
{
UserAgentSuffix = String.Format(CultureInfo.InvariantCulture, Resources.CustomUserAgentSuffixFormat,
entryAssembly == null ? Resources.UnknownEntryAssembly : entryAssembly.GetName().Name,
Assembly.GetExecutingAssembly().GetName().Version,
context.SourceName, context.SinkName,
isShardedImport ? Resources.ShardedImportDesignator : String.Empty)
};
RetryOptions retryOptions = new RetryOptions();
if (retries.HasValue)
{
retryOptions.MaxRetryAttemptsOnThrottledRequests = retries.Value;
}
if (retryInterval.HasValue)
{
retryOptions.MaxRetryWaitTimeInSeconds = retryInterval.Value.Seconds;
}
connectionPolicy.RetryOptions = retryOptions;
if (maxConnectionLimit.HasValue)
{
connectionPolicy.MaxConnectionLimit = maxConnectionLimit.Value;
}
return(DocumentDbClientHelper.ApplyConnectionMode(connectionPolicy, connectionMode));
}
public Task <IDataSinkAdapter> CreateAsync(TConfiguration configuration, IDataTransferContext context)
{
ValidateBaseConfiguration(configuration);
return(CreateAsync(CreateClient(configuration, context), GetDataItemTransformation(configuration), configuration));
}
private static DocumentClient CreateRawClient(IDocumentDbConnectionSettings connectionSettings, DocumentDbConnectionMode?connectionMode, IDataTransferContext context,
bool isShardedImport, int?maxConnectionLimit, int?retries, TimeSpan?retryInterval, bool?ignoreSSLCertErrors = false)
{
Guard.NotNull("connectionSettings", connectionSettings);
return(new DocumentClient(
new Uri(connectionSettings.AccountEndpoint),
connectionSettings.AccountKey, ignoreSSLCertErrors.GetValueOrDefault() ? DocumentDbClientHelper.GetSSLCertHandler() : null, !ignoreSSLCertErrors.GetValueOrDefault() ?
CreateConnectionPolicy(connectionMode, context, isShardedImport, maxConnectionLimit, retries, retryInterval) : null
));
}
