public workerThread(
SqlCommand comm,
roundRobinBuffer outputRows,
RowLogicDelegate rowLogic,
bool reuseConnection,
int packageSize,
roundRobinBuffer bulkRows,
bool impersonateCaller,
IntPtr callerIdentityToken,
threadMonitor mainThreadMonitor)
{
this.comm = comm;
this.outputRows = outputRows;
this.rowLogic = rowLogic;
this.reuseConnection = reuseConnection;
this.packageSize = packageSize;
this.bulkRows = bulkRows;
this.impersonateCaller = impersonateCaller;
this.callerIdentityToken = callerIdentityToken;
this.mainThreadMonitor = mainThreadMonitor;
Thread t = new Thread(new ThreadStart(this.doWork));
//t.Name = "QueryParallelizer: Worker Thread (" + comm.CommandText.Substring(0, 50) + ")";
this.worker = t;
}
/// <summary>
/// Constructor for the QueryParallelizer class.
/// </summary>
/// <param name="query">
/// The query that will be called by the workers. The query should be formatted such that it contains
/// a predicate (e.g. BETWEEN) that operates on an integer range (or some other data type converted
/// from an integer). The predicate should use placeholder variables, which are specified using the
/// minVariable and maxVariable parameters. Any given integer subrange should return a unique set of
/// rows in order to ensure that the output of the entire operation is a unique set of rows.
/// </param>
/// <param name="minVariable">
/// The name of the placeholder variable used for the minimum value in the query's integer range.
/// </param>
/// <param name="maxVariable">
/// The name of the placeholder variable used for the maximum value in the query's integer range.
/// </param>
/// <param name="minValue">
/// The minimum value in the entire integer range to be processed by the QueryParallelizer.
/// </param>
/// <param name="maxValue">
/// The maximum value in the entire integer range to be processed by the QueryParallelizer.
/// </param>
/// <param name="numWorkerThreads">
/// The number of workers that should be used to process the data in the integer range. Note that if
/// the range has fewer numbers than the number of workers specified here, or if the BatchSize property
/// is set such that fewer sub-ranges will be produced than the number specified here, this value will
/// be internally overridden in order to avoid creating more workers than are necssary.
/// </param>
/// <param name="rowLogic">
/// The delegate to be used for row manipulation logic.
/// </param>
public QueryParallelizer(
string query,
string minVariable,
string maxVariable,
int minValue,
int maxValue,
int numWorkerThreads,
RowLogicDelegate rowLogic)
{
if (numWorkerThreads < 1 || numWorkerThreads > 32)
{
throw new ArgumentException("Number of worker threads must be between 1 and 32");
}
long totalRange = (long)maxValue - (long)minValue + 1;
if ((long)numWorkerThreads > totalRange)
{
numWorkerThreads = (int)totalRange;
}
if (minValue > maxValue)
{
throw new ArgumentException("MinValue must be less than or equal to MaxValue.");
}
this.query = query;
this.minVariable = minVariable;
this.maxVariable = maxVariable;
this.minValue = minValue;
this.maxValue = maxValue;
this.numWorkerThreads = numWorkerThreads;
this.rowLogic = rowLogic;
targetConnectionString = ConnectionBuilder.LoopbackConnectionString;
}
public internalEnumerator(
string query,
string minVariable,
string maxVariable,
int minValue,
int maxValue,
int numWorkerThreads,
RowLogicDelegate rowLogic,
string targetConnectionString,
int rowBufferSize,
int packageSize,
int batchSize,
bool reuseConnection,
BulkCopySettings bulkSettings,
bool impersonateCaller,
IntPtr callerIdentityToken,
SqlParameter[] additionalParameters)
{
try
{
Thread.BeginThreadAffinity();
threadMonitor mainThreadMonitor = new threadMonitor(System.Threading.Thread.CurrentThread);
outputRows = new roundRobinBuffer(rowBufferSize);
completionEvent = new ManualResetEvent(false);
//buffer that will be shared by the task workers and the bulk workers
roundRobinBuffer bulkRows = null;
//event that will be shared by the monitor thread and the bulk workers
ManualResetEvent bulkCompletionEvent = new ManualResetEvent(false);
bool outputRowsToExternalEnumerator = true;
List <bulkThread> bulkWorkers = new List <bulkThread>();
if (bulkSettings != null)
{
outputRowsToExternalEnumerator = bulkSettings.OutputRowsToExternalEnumerator;
bulkRows = new roundRobinBuffer(rowBufferSize);
for (int i = 1; i <= bulkSettings.NumThreads; i++)
{
bulkWorkers.Add(
new bulkThread(
bulkSettings.ConnectionString,
bulkSettings.DestinationTable,
bulkSettings.CopyOptions,
bulkSettings.BatchSize,
bulkSettings.ColumnMap,
bulkSettings.FieldCount,
bulkSettings.DataMap,
bulkRows,
bulkCompletionEvent,
impersonateCaller,
callerIdentityToken,
mainThreadMonitor));
}
}
List <workerThread> workers = new List <workerThread>();
List <workPackage> workPackages = new List <workPackage>();
int minValueThread;
int maxValueThread;
long range = ((long)maxValue - (long)minValue + 1);
//initialize the worker threads
if (batchSize <= 0)
{
//if there are fewer threads than values in the range, do a simple one-to-one distribution
if (range < numWorkerThreads)
{
for (int i = minValue; i <= maxValue; i++)
{
workPackages.Add(new workPackage(i, i));
}
}
else
{
for (int i = 0; i < numWorkerThreads; i++)
{
//determine the correct min and max values per thread
minValueThread = (int)((long)minValue + ((range / (long)numWorkerThreads) * (long)i));
maxValueThread = ((i + 1) == numWorkerThreads) ? maxValue : (int)((long)minValue - 1 + ((range / (long)numWorkerThreads) * (long)(i + 1)));
//at edges we may have more threads than can be used - ignore them
if (maxValueThread < int.MaxValue && minValueThread > int.MinValue)
{
if (maxValueThread == (minValueThread - 1))
{
numWorkerThreads--;
continue;
}
}
workPackages.Add(new workPackage(minValueThread, maxValueThread));
}
}
}
else
{
minValueThread = minValue;
maxValueThread = minValueThread + (batchSize - 1);
do
{
if (maxValueThread > maxValue)
{
maxValueThread = maxValue;
}
workPackages.Add(new workPackage(minValueThread, maxValueThread));
if (maxValueThread == maxValue)
{
break;
}
int tempMaxValue = maxValueThread;
minValueThread = maxValueThread + 1;
maxValueThread = minValueThread + (batchSize - 1);
//has an overflow occurred?
if (maxValueThread < tempMaxValue)
{
maxValueThread = maxValue;
}
} while (minValueThread <= maxValue);
}
for (int j = 0; j < ((workPackages.Count < numWorkerThreads) ? workPackages.Count : numWorkerThreads); j++)
{
SqlConnection conn = new SqlConnection((targetConnectionString == null) ? ConnectionBuilder.LoopbackConnectionString : targetConnectionString);
SqlCommand comm = conn.CreateCommand();
comm.CommandText = query;
comm.CommandTimeout = 0;
comm.Parameters.Add(minVariable, System.Data.SqlDbType.Int);
comm.Parameters.Add(maxVariable, System.Data.SqlDbType.Int);
if (additionalParameters != null)
{
foreach (SqlParameter param in additionalParameters)
{
comm.Parameters.Add(param);
}
}
//spin up worker thread, pass in the required args
workers.Add(
new workerThread(
comm,
outputRowsToExternalEnumerator ? outputRows : null,
rowLogic,
reuseConnection,
packageSize,
bulkRows,
impersonateCaller,
callerIdentityToken,
mainThreadMonitor));
}
//Event that will be signaled when the monitor is initialized
ManualResetEvent initializationEvent = new ManualResetEvent(false);
//initialize the monitor thread
mp = new monitorThread(
mainThreadMonitor,
initializationEvent,
completionEvent,
bulkCompletionEvent,
workers.ToArray(),
bulkWorkers.ToArray(),
workPackages.ToArray());
var t = new Thread(new ThreadStart(mp.SetupMonitorThread));
//t.Name = "QueryParallelizer: Monitor Thread";
//start the monitor thread
t.Start();
//Wait up to 20 seconds + [number of workers] seconds * 2 for initialization
int waitInterval = 20000 + ((workers.Count + bulkWorkers.Count) * 2000);
bool initialized = initializationEvent.WaitOne(waitInterval);
if (mp.WorkerThreadException != null)
{
throw mp.WorkerThreadException;
}
if (!initialized)
{
throw new Exception(String.Format("Monitor thread failed to initialize within {0} milliseconds", waitInterval));
}
}
finally
{
Thread.EndThreadAffinity();
}
}
