private List <CopyInfo> CreateLogicalPartitionedTableCopyInfo(string tableName)
{
_logger.Info($"Table {tableName} is NOT partitioned. Bulk copy will be parallelized using {_config.LogicalPartitions} logical partitions.");
var copyInfo = new List <CopyInfo>();
foreach (var n in Enumerable.Range(1, _config.LogicalPartitions))
{
var cp = new LogicalPartitionCopyInfo();
cp.PartitionNumber = n;
cp.TableName = tableName;
cp.LogicalPartitionsCount = _config.LogicalPartitions;
copyInfo.Add(cp);
}
return(copyInfo);
}
private List <CopyInfo> CreateLogicalPartitionedTableCopyInfo(TableInfo ti)
{
string tableName = ti.TableName;
TableSize tableSize = ti.Size;
var copyInfo = new List <CopyInfo>();
long partitionCount = 1;
_logger.Debug($"{tableName}: RowCount={tableSize.RowCount}, SizeInGB={tableSize.SizeInGB}");
switch (_config.LogicalPartitioningStrategy)
{
case LogicalPartitioningStrategy.Auto:
// One partition per GB
partitionCount = tableSize.SizeInGB;
// If table is small in size but has a lot of rows
if (tableSize.SizeInGB < 1 && tableSize.RowCount > _config.BatchSize)
{
partitionCount = tableSize.RowCount / (_config.BatchSize * 10);
}
var maxPartitions = _config.MaxParallelTasks * 3;
if (partitionCount < 3)
{
partitionCount = 3;
}
if (partitionCount > maxPartitions)
{
partitionCount = maxPartitions;
}
break;
case LogicalPartitioningStrategy.Size:
partitionCount = tableSize.SizeInGB / _config.LogicalPartitions;
break;
case LogicalPartitioningStrategy.Count:
partitionCount = _config.LogicalPartitions;
break;
}
if (partitionCount % 2 == 0)
{
partitionCount += 1; // Make sure number is odd.
}
var ps = (double)tableSize.SizeInGB / (double)partitionCount;
var pc = (double)tableSize.RowCount / (double)partitionCount;
_logger.Info($"{tableName} -> Parallel load will use {partitionCount} logical partitions (Logical partition size: {ps:0.00} GB, Rows: {pc:0.00}).");
foreach (var n in Enumerable.Range(1, (int)partitionCount))
{
var cp = new LogicalPartitionCopyInfo();
cp.PartitionNumber = n;
cp.SourceTableInfo = ti;
cp.LogicalPartitionsCount = (int)partitionCount;
copyInfo.Add(cp);
}
return(copyInfo);
}
