Exemplo n.º 1
0
        private async Task CopyDataAsync(bool sampleData = false, SqlPipelines pipelines = null)
        {
            _logger.LogInformation("Copying data started...");
            _primaryKeyNames = new Dictionary <string, List <string> >();

            var tables = (await _sqlConnection.QueryAsync(@"
                select SCHEMA_NAME(t.schema_id) AS TABLE_SCHEMA, t.name AS TABLE_NAME
                from sys.tables t
                where t.temporal_type_desc IN ('SYSTEM_VERSIONED_TEMPORAL_TABLE', 'NON_TEMPORAL_TABLE')
            ")).ToList();

            // *** refresh to workaround NCBC-2784
            Dispose();
            await ConnectAsync();
            await CreateBucketAsync();

            // ****

            foreach (var table in tables)
            {
                await CopyDataFromTableToCollectionAsync(table.TABLE_SCHEMA, table.TABLE_NAME, sampleData, pipelines);
            }

            _logger.LogInformation("Copying data complete.");
        }
Exemplo n.º 2
0
        /// <summary>
        /// Start one or more Migration processes
        /// </summary>
        /// <param name="validateNames">Check to make sure the table names are short enough or are mapped properly</param>
        /// <param name="createBucket">Create a Couchbase bucket to match the SQL catalog</param>
        /// <param name="createCollections">Create Couchbase collections to match the SQL tables</param>
        /// <param name="createIndexes">Create Couchbase indexes to match the SQL indexes</param>
        /// <param name="copyData">Create Couchbase documents to match the SQL rows</param>
        /// <param name="sampleForDemo">Only create a sample set of Couchbase documents/indexes - only suitable for demos!</param>
        /// <param name="createUsers">Create Couchbase users to match the SQL users</param>
        /// <param name="pipelines">Pipelines mapped to tables to perform filters/transforms</param>
        /// <returns></returns>
        public async Task MigrateAsync(
            bool validateNames     = false,
            bool createBucket      = false,
            bool createCollections = false,
            bool createIndexes     = false,
            bool copyData          = false,
            bool sampleForDemo     = false,
            bool createUsers       = false,
            SqlPipelines pipelines = null)
        {
            if (validateNames)
            {
                await ValidateNamesAsync();
            }

            var shouldContinue = createBucket || createCollections || createIndexes || copyData || createUsers;

            if (!shouldContinue)
            {
                return;
            }

            if (!_isValid)
            {
                throw new Exception("Validation on SQL Server names has not been performed. Cannot continue with migration.");
            }

            if (createBucket)
            {
                await CreateBucketAsync();
            }
            else
            {
                await ConnectBucketAsync();
            }

            if (createCollections)
            {
                await CreateCollectionsAsync();
            }

            if (createIndexes)
            {
                await CreateIndexesAsync(sampleForDemo);
            }

            if (copyData)
            {
                await CopyDataAsync(sampleForDemo, pipelines);
            }

            if (createUsers)
            {
                await CreateUsersAsync();
            }
        }
Exemplo n.º 3
0
        private async Task CopyDataFromTableToCollectionAsync(string tableSchema, string tableName, bool sampleData = false, SqlPipelines pipelines = null)
        {
            var collectionName = GetCollectionName(tableSchema, tableName);

            _logger.LogInformation($"Copying data from `{tableSchema}.{tableName}` table to {collectionName} collection...");
            var counter = 0;

            ICouchbaseCollection collection;

            if (_config.UseSchemaForScope)
            {
                var scopeName = GetScopeName(tableSchema);
                var scope     = await _bucket.ScopeAsync(scopeName);

                collection = await scope.CollectionAsync(collectionName);
            }
            else
            {
                var scope = await _bucket.DefaultScopeAsync();

                collection = await scope.CollectionAsync(collectionName);
            }

            // lookup and use custom pipeline defined for this table
            SqlPipelineBase pipeline = new SqlPipelineDefault(tableSchema, tableName);

            if (sampleData)
            {
                pipeline = new SqlPipelineDefaultSample(tableSchema, tableName);
            }
            var customPipeline = pipelines?.Get(tableSchema, tableName);

            if (customPipeline != null)
            {
                pipeline = customPipeline;
            }

            // buffered false because this could be a very large amount of data
            // see: https://dapper-tutorial.net/buffered
            using (var outerConnection = new SqlConnection(_config.SourceSqlConnectionString))
            {
                _logger.LogInformation($"Opening non-buffered connection to `{tableSchema}.{tableName}`");
                var rows = outerConnection.Query(pipeline.Query, buffered: false);

                _logger.LogInformation("Writing row(s)...");
                foreach (var row in rows)
                {
                    if (!pipeline.IsIncluded(row))
                    {
                        continue;
                    }

                    var transformedRow = pipeline.Transform(row);

                    string documentKey = null;
                    try
                    {
                        documentKey = await GetDocumentKeyFromPrimaryKeyValuesAsync(transformedRow, tableSchema, tableName);

                        await collection.UpsertAsync(documentKey, transformedRow);
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError($"Error writing.");
                        _logger.LogError($"Row data: {transformedRow}");
                        _logger.LogError($"Document key: {documentKey}");
                        _logger.LogError($"Exception message: {ex.Message}");
                        _logger.LogError($"Exception stack trace: {ex.StackTrace}");
                    }

                    counter++;
                    if ((counter % 1000) == 0)
                    {
                        _logger.LogInformation(counter + "...");
                    }
                }
            }
            _logger.LogInformation($"Closing non-buffered connection to `{tableSchema}.{tableName}`");
            _logger.LogInformation("Done");
        }