public void CreateChunks()
{
dbKeyOffset.Create(Settings.Current.Building.Id.Value);
dbChunk.ClearChunks(Settings.Current.Building.Id.Value);
dbSource.CreateChunkTable();
foreach (var chunk in GetPersonKeys(Settings.Current.Builder.BatchSize))
{
var chunkId = dbChunk.AddChunk(Settings.Current.Building.Id.Value);
using (var bulkCopy = new SqlBulkCopy(Settings.Current.Building.SourceConnectionString))
{
bulkCopy.Write(
new ChunkDataReader(
chunk.Select(
c => new ChunkRecord {
Id = chunkId, PersonId = Convert.ToInt64(c.Key), PersonSource = c.Value
})
.ToList()),
"_chunks");
bulkCopy.Close();
}
}
dbSource.CreateIndexesChunkTable();
}
public void CreateChunks()
{
//TotalPersonCount = 0;
var chunkIds = new List <int>();
dbKeyOffset.Recreate(Settings.Current.Building.Id.Value);
dbChunk.ClearChunks(Settings.Current.Building.Id.Value);
dbSource.CreateChunkTable();
int i = 0;
int k = 0;
using (var saver = Settings.Current.Building.SourceEngine.GetSaver().Create(Settings.Current.Building.SourceConnectionString))
{
var chunks = new List <ChunkRecord>();
foreach (var chunk in GetPersonKeys(Settings.Current.Building.BatchSize))
{
//TotalPersonCount = TotalPersonCount + chunk.Count;
var chunkId = dbChunk.AddChunk(Settings.Current.Building.Id.Value);
chunkIds.Add(chunkId);
//TMP
//if (Settings.Current.Building.SourceEngine.Database == Database.MSSQL)
// AddSubChunks(chunkId, chunk);
chunks.AddRange(chunk.Select(c => new ChunkRecord {
Id = chunkId, PersonId = Convert.ToInt64(c.Key), PersonSource = c.Value
}));
var chunkSizeOnS3 = 0;
if (Settings.Current.Building.BatchSize >= 10000 && Settings.Current.Building.BatchSize < 500000)
{
chunkSizeOnS3 = 200;
}
else if (Settings.Current.Building.BatchSize >= 500000 && Settings.Current.Building.BatchSize < 2000000)
{
chunkSizeOnS3 = 20;
}
else if (Settings.Current.Building.BatchSize >= 2000000)
{
chunkSizeOnS3 = 2;
}
if (Settings.Current.Building.SourceEngine.Database == Database.Redshift &&
Settings.Current.Building.BatchSize >= 10000 && i == chunkSizeOnS3)
{
saver.AddChunk(chunks, k);
chunks.Clear();
i = 0;
k++;
}
i++;
}
if (chunks.Count > 0)
{
saver.AddChunk(chunks, k);
}
saver.Commit();
}
dbSource.CreateIndexesChunkTable();
if (Settings.Current.Building.SourceEngine.Database == Database.Redshift)
{
MoveChunkDataToS3(chunkIds);
try
{
dbSource.GrantAccessToChunkTable();
}
catch (Exception e)
{
Logger.Write(null, LogMessageTypes.Warning, "GrantAccessToChunkTable " + Logger.CreateExceptionString(e));
}
}
}
public void CreateChunks(bool restart)
{
if (restart && Settings.Settings.Current.Building.SourceEngine.Database != Database.Redshift)
{
restart = false;
}
Console.WriteLine("Source connection string=" + Settings.Settings.Current.Building.SourceConnectionString);
if (!restart)
{
Console.WriteLine("Generating ids...");
_dbChunk.ClearChunks(Settings.Settings.Current.Building.Id.Value);
_dbSource.CreateChunkTable();
if (Settings.Settings.Current.Building.SourceEngine.Database == Database.Redshift)
{
try
{
_dbSource.GrantAccessToChunkTable();
}
catch (Exception e)
{
Logger.Write(null, LogMessageTypes.Warning,
"GrantAccessToChunkTable " + Logger.CreateExceptionString(e));
}
}
var chunkId = 0;
//int i = 0;
int k = 0;
//var chunkSizeOnS3 = 0;
//if (Settings.Settings.Current.Building.BatchSize >= 10000 && Settings.Settings.Current.Building.BatchSize < 500000)
// chunkSizeOnS3 = 50;
//else if (Settings.Settings.Current.Building.BatchSize >= 500000 && Settings.Settings.Current.Building.BatchSize < 2000000)
// chunkSizeOnS3 = 5;
//else if (Settings.Settings.Current.Building.BatchSize >= 2000000)
// chunkSizeOnS3 = 1;
using (var saver = Settings.Settings.Current.Building.SourceEngine.GetSaver().Create(Settings.Settings.Current.Building.SourceConnectionString))
{
var chunks = new List <ChunkRecord>();
foreach (var chunk in GetPersonKeys(Settings.Settings.Current.Building.BatchSize))
{
_dbChunk.AddChunk(chunkId, Settings.Settings.Current.Building.Id.Value);
chunks.AddRange(chunk.Select(c => new ChunkRecord {
Id = chunkId, PersonId = Convert.ToInt64(c.Key), PersonSource = c.Value
}));
if (Settings.Settings.Current.Building.SourceEngine.Database == Database.Redshift)
//if (Settings.Settings.Current.Building.SourceEngine.Database == Database.Redshift &&
// Settings.Settings.Current.Building.BatchSize >= 10000 && i == chunkSizeOnS3)
{
saver.AddChunk(chunks, k);
chunks.Clear();
GC.Collect();
//i = 0;
k++;
}
//i++;
chunkId++;
}
if (chunks.Count > 0)
{
saver.AddChunk(chunks, k);
}
saver.Commit();
}
Console.WriteLine("chunk ids were generated and saved, total count=" + chunkId);
}
if (Settings.Settings.Current.Building.SourceEngine.Database == Database.Redshift)
{
Console.WriteLine("Moving raw data to S3...");
MoveChunkDataToS3();
Console.WriteLine("Moving data - complete");
}
}