private async Task CopyDataFromTableToCollectionAsync(string tableSchema, string tableName, bool sampleData = false, SqlPipelines pipelines = null)
{
var collectionName = GetCollectionName(tableSchema, tableName);
_logger.LogInformation($"Copying data from `{tableSchema}.{tableName}` table to {collectionName} collection...");
var counter = 0;
ICouchbaseCollection collection;
if (_config.UseSchemaForScope)
{
var scopeName = GetScopeName(tableSchema);
var scope = await _bucket.ScopeAsync(scopeName);
collection = await scope.CollectionAsync(collectionName);
}
else
{
var scope = await _bucket.DefaultScopeAsync();
collection = await scope.CollectionAsync(collectionName);
}
// lookup and use custom pipeline defined for this table
SqlPipelineBase pipeline = new SqlPipelineDefault(tableSchema, tableName);
if (sampleData)
{
pipeline = new SqlPipelineDefaultSample(tableSchema, tableName);
}
var customPipeline = pipelines?.Get(tableSchema, tableName);
if (customPipeline != null)
{
pipeline = customPipeline;
}
// buffered false because this could be a very large amount of data
// see: https://dapper-tutorial.net/buffered
using (var outerConnection = new SqlConnection(_config.SourceSqlConnectionString))
{
_logger.LogInformation($"Opening non-buffered connection to `{tableSchema}.{tableName}`");
var rows = outerConnection.Query(pipeline.Query, buffered: false);
_logger.LogInformation("Writing row(s)...");
foreach (var row in rows)
{
if (!pipeline.IsIncluded(row))
{
continue;
}
var transformedRow = pipeline.Transform(row);
string documentKey = null;
try
{
documentKey = await GetDocumentKeyFromPrimaryKeyValuesAsync(transformedRow, tableSchema, tableName);
await collection.UpsertAsync(documentKey, transformedRow);
}
catch (Exception ex)
{
_logger.LogError($"Error writing.");
_logger.LogError($"Row data: {transformedRow}");
_logger.LogError($"Document key: {documentKey}");
_logger.LogError($"Exception message: {ex.Message}");
_logger.LogError($"Exception stack trace: {ex.StackTrace}");
}
counter++;
if ((counter % 1000) == 0)
{
_logger.LogInformation(counter + "...");
}
}
}
_logger.LogInformation($"Closing non-buffered connection to `{tableSchema}.{tableName}`");
_logger.LogInformation("Done");
}