Example #1
0
        public void CreateChunks()
        {
            dbKeyOffset.Create(Settings.Current.Building.Id.Value);
            dbChunk.ClearChunks(Settings.Current.Building.Id.Value);
            dbSource.CreateChunkTable();
            foreach (var chunk in GetPersonKeys(Settings.Current.Builder.BatchSize))
            {
                var chunkId = dbChunk.AddChunk(Settings.Current.Building.Id.Value);

                using (var bulkCopy = new SqlBulkCopy(Settings.Current.Building.SourceConnectionString))
                {
                    bulkCopy.Write(
                        new ChunkDataReader(
                            chunk.Select(
                                c => new ChunkRecord {
                        Id = chunkId, PersonId = Convert.ToInt64(c.Key), PersonSource = c.Value
                    })
                            .ToList()),
                        "_chunks");
                    bulkCopy.Close();
                }
            }
            dbSource.CreateIndexesChunkTable();
        }
Example #2
0
        public void CreateChunks()
        {
            //TotalPersonCount = 0;
            var chunkIds = new List <int>();

            dbKeyOffset.Recreate(Settings.Current.Building.Id.Value);
            dbChunk.ClearChunks(Settings.Current.Building.Id.Value);
            dbSource.CreateChunkTable();
            int i = 0;
            int k = 0;

            using (var saver = Settings.Current.Building.SourceEngine.GetSaver().Create(Settings.Current.Building.SourceConnectionString))
            {
                var chunks = new List <ChunkRecord>();
                foreach (var chunk in GetPersonKeys(Settings.Current.Building.BatchSize))
                {
                    //TotalPersonCount = TotalPersonCount + chunk.Count;

                    var chunkId = dbChunk.AddChunk(Settings.Current.Building.Id.Value);
                    chunkIds.Add(chunkId);

                    //TMP
                    //if (Settings.Current.Building.SourceEngine.Database == Database.MSSQL)
                    //   AddSubChunks(chunkId, chunk);

                    chunks.AddRange(chunk.Select(c => new ChunkRecord {
                        Id = chunkId, PersonId = Convert.ToInt64(c.Key), PersonSource = c.Value
                    }));
                    var chunkSizeOnS3 = 0;
                    if (Settings.Current.Building.BatchSize >= 10000 && Settings.Current.Building.BatchSize < 500000)
                    {
                        chunkSizeOnS3 = 200;
                    }
                    else if (Settings.Current.Building.BatchSize >= 500000 && Settings.Current.Building.BatchSize < 2000000)
                    {
                        chunkSizeOnS3 = 20;
                    }
                    else if (Settings.Current.Building.BatchSize >= 2000000)
                    {
                        chunkSizeOnS3 = 2;
                    }

                    if (Settings.Current.Building.SourceEngine.Database == Database.Redshift &&
                        Settings.Current.Building.BatchSize >= 10000 && i == chunkSizeOnS3)
                    {
                        saver.AddChunk(chunks, k);
                        chunks.Clear();
                        i = 0;
                        k++;
                    }
                    i++;
                }
                if (chunks.Count > 0)
                {
                    saver.AddChunk(chunks, k);
                }

                saver.Commit();
            }
            dbSource.CreateIndexesChunkTable();

            if (Settings.Current.Building.SourceEngine.Database == Database.Redshift)
            {
                MoveChunkDataToS3(chunkIds);
                try
                {
                    dbSource.GrantAccessToChunkTable();
                }
                catch (Exception e)
                {
                    Logger.Write(null, LogMessageTypes.Warning, "GrantAccessToChunkTable " + Logger.CreateExceptionString(e));
                }
            }
        }
Example #3
0
        public void CreateChunks(bool restart)
        {
            if (restart && Settings.Settings.Current.Building.SourceEngine.Database != Database.Redshift)
            {
                restart = false;
            }

            Console.WriteLine("Source connection string=" + Settings.Settings.Current.Building.SourceConnectionString);

            if (!restart)
            {
                Console.WriteLine("Generating ids...");
                _dbChunk.ClearChunks(Settings.Settings.Current.Building.Id.Value);
                _dbSource.CreateChunkTable();

                if (Settings.Settings.Current.Building.SourceEngine.Database == Database.Redshift)
                {
                    try
                    {
                        _dbSource.GrantAccessToChunkTable();
                    }
                    catch (Exception e)
                    {
                        Logger.Write(null, LogMessageTypes.Warning,
                                     "GrantAccessToChunkTable " + Logger.CreateExceptionString(e));
                    }
                }

                var chunkId = 0;
                //int i = 0;
                int k = 0;

                //var chunkSizeOnS3 = 0;
                //if (Settings.Settings.Current.Building.BatchSize >= 10000 && Settings.Settings.Current.Building.BatchSize < 500000)
                //    chunkSizeOnS3 = 50;
                //else if (Settings.Settings.Current.Building.BatchSize >= 500000 && Settings.Settings.Current.Building.BatchSize < 2000000)
                //    chunkSizeOnS3 = 5;
                //else if (Settings.Settings.Current.Building.BatchSize >= 2000000)
                //    chunkSizeOnS3 = 1;

                using (var saver = Settings.Settings.Current.Building.SourceEngine.GetSaver().Create(Settings.Settings.Current.Building.SourceConnectionString))
                {
                    var chunks = new List <ChunkRecord>();
                    foreach (var chunk in GetPersonKeys(Settings.Settings.Current.Building.BatchSize))
                    {
                        _dbChunk.AddChunk(chunkId, Settings.Settings.Current.Building.Id.Value);

                        chunks.AddRange(chunk.Select(c => new ChunkRecord {
                            Id = chunkId, PersonId = Convert.ToInt64(c.Key), PersonSource = c.Value
                        }));

                        if (Settings.Settings.Current.Building.SourceEngine.Database == Database.Redshift)
                        //if (Settings.Settings.Current.Building.SourceEngine.Database == Database.Redshift &&
                        //    Settings.Settings.Current.Building.BatchSize >= 10000 && i == chunkSizeOnS3)
                        {
                            saver.AddChunk(chunks, k);
                            chunks.Clear();
                            GC.Collect();
                            //i = 0;
                            k++;
                        }
                        //i++;
                        chunkId++;
                    }
                    if (chunks.Count > 0)
                    {
                        saver.AddChunk(chunks, k);
                    }

                    saver.Commit();
                }
                Console.WriteLine("chunk ids were generated and saved, total count=" + chunkId);
            }

            if (Settings.Settings.Current.Building.SourceEngine.Database == Database.Redshift)
            {
                Console.WriteLine("Moving raw data to S3...");
                MoveChunkDataToS3();
                Console.WriteLine("Moving data - complete");
            }
        }