public void CreateChunks()
{
//TotalPersonCount = 0;
var chunkIds = new List <int>();
dbKeyOffset.Recreate(Settings.Current.Building.Id.Value);
dbChunk.ClearChunks(Settings.Current.Building.Id.Value);
dbSource.CreateChunkTable();
int i = 0;
int k = 0;
using (var saver = Settings.Current.Building.SourceEngine.GetSaver().Create(Settings.Current.Building.SourceConnectionString))
{
var chunks = new List <ChunkRecord>();
foreach (var chunk in GetPersonKeys(Settings.Current.Building.BatchSize))
{
//TotalPersonCount = TotalPersonCount + chunk.Count;
var chunkId = dbChunk.AddChunk(Settings.Current.Building.Id.Value);
chunkIds.Add(chunkId);
//TMP
//if (Settings.Current.Building.SourceEngine.Database == Database.MSSQL)
// AddSubChunks(chunkId, chunk);
chunks.AddRange(chunk.Select(c => new ChunkRecord {
Id = chunkId, PersonId = Convert.ToInt64(c.Key), PersonSource = c.Value
}));
var chunkSizeOnS3 = 0;
if (Settings.Current.Building.BatchSize >= 10000 && Settings.Current.Building.BatchSize < 500000)
{
chunkSizeOnS3 = 200;
}
else if (Settings.Current.Building.BatchSize >= 500000 && Settings.Current.Building.BatchSize < 2000000)
{
chunkSizeOnS3 = 20;
}
else if (Settings.Current.Building.BatchSize >= 2000000)
{
chunkSizeOnS3 = 2;
}
if (Settings.Current.Building.SourceEngine.Database == Database.Redshift &&
Settings.Current.Building.BatchSize >= 10000 && i == chunkSizeOnS3)
{
saver.AddChunk(chunks, k);
chunks.Clear();
i = 0;
k++;
}
i++;
}
if (chunks.Count > 0)
{
saver.AddChunk(chunks, k);
}
saver.Commit();
}
dbSource.CreateIndexesChunkTable();
if (Settings.Current.Building.SourceEngine.Database == Database.Redshift)
{
MoveChunkDataToS3(chunkIds);
try
{
dbSource.GrantAccessToChunkTable();
}
catch (Exception e)
{
Logger.Write(null, LogMessageTypes.Warning, "GrantAccessToChunkTable " + Logger.CreateExceptionString(e));
}
}
}
