public void TestThreadedConcurrentStackedBagSlim()
{
const int loopCount = 50000;
var done = false;
var sum = 0;
var bag = new ConcurrentStackedBagSlim <int>();
var threadInserter = new Thread(_ =>
{
for (var i = 1; i <= loopCount; i++)
{
bag.Add(i);
}
done = true;
});
threadInserter.Start();
_parallel.For(0, loopCount, i =>
{
while (true)
{
if (done && bag.IsEmpty)
{
break;
}
if (bag.TryTake(out var n))
{
Interlocked.Add(ref sum, n);
}
}
public void TestBoundedParallelArgumentsCheck()
{
var boundedParallel = new BoundedParallel();
ParallelOptions nullParallelOptions = null;
Assert.IsTrue(Assert.ThrowsException <ArgumentNullException>(() => boundedParallel.Invoke(nullParallelOptions, delegate { })).Message.Contains("parallelOptions"));
Assert.IsTrue(Assert.ThrowsException <ArgumentNullException>(() => boundedParallel.Invoke(new ParallelOptions(), null)).Message.Contains("actions"));
Assert.IsTrue(Assert.ThrowsException <ArgumentException>(() => boundedParallel.Invoke(new ParallelOptions(), new Action[] { null })).Message.Contains("null action"));
Assert.IsTrue(Assert.ThrowsException <ArgumentNullException>(() => boundedParallel.For(0, 1, nullParallelOptions, delegate { })).Message.Contains("parallelOptions"));
Assert.IsTrue(Assert.ThrowsException <ArgumentNullException>(() => boundedParallel.For(0, 1, new ParallelOptions(), null)).Message.Contains("body"));
IEnumerable <int> nullEnumerable = null;
Assert.IsTrue(Assert.ThrowsException <ArgumentNullException>(() => boundedParallel.ForEach(nullEnumerable, new ParallelOptions(), delegate { })).Message.Contains("source"));
Assert.IsTrue(Assert.ThrowsException <ArgumentNullException>(() => boundedParallel.ForEach(new List <int>(), null, delegate { })).Message.Contains("parallelOptions"));
Assert.IsTrue(Assert.ThrowsException <ArgumentNullException>(() => boundedParallel.ForEach(new List <int>(), new ParallelOptions(), null)).Message.Contains("body"));
}
public virtual void Prepare()
{
if (_prepared)
{
throw new InvalidOperationException("DataReplicator already prepared.");
}
try
{
var sqlBuilder = (IClassSqlBuilder)typeof(TSourceAdapter);
_sourceCmds = new List <DbCommand>();
foreach (var sqlStatement in _sourceTables)
{
_sourceCmds.Add(sqlStatement.Item4 ?? sqlBuilder.BuildCommand(sqlStatement.Item2, null));
sqlStatement.Item4 = null;
}
_sourceConnections = new DbConnection[_sourceCmds.Count];
_targetConnections = new TTargetConn[_sourceCmds.Count];
ColumnMetadataLists = new ColumnMetadataList[_sourceCmds.Count];
_readers = new List <DbDataReader> {
Capacity = _sourceCmds.Count
};
for (var i = 0; i < _sourceCmds.Count; i++)
{
_readers.Add(null);
}
_parallelRunner = new BoundedParallel(1, ParallelismLevel);
_parallelRunner.For(0, _sourceCmds.Count, i =>
{
if (_sourceTables[i].Item4 != null)
{
_sourceConnections[i] = _sourceTables[i].Item4.Connection;
}
else
{
_sourceConnections[i] = sqlBuilder.BuildConnection(_sourceConnStr);
}
_sourceConnections[i].Open();
_sourceCmds[i].Connection = _sourceConnections[i];
var reader = _sourceCmds[i].ExecuteReader();
_readers[i] = reader;
ColumnMetadataLists[i] = new ColumnMetadataListFromDbReader(reader);
_targetConnections[i] = GenericObjectBuilder.Build <TTargetConn>(_targetConnStr);
_targetConnections[i].Open();
});
_prepared = true;
}
catch (Exception)
{
UnPrepare(true);
throw;
}
}
public void TestBoundedParallelSimpleForWithZeroItems()
{
var regularParallelResult = Parallel.For(0, 0, (n) => { });
var boundedParallel = new BoundedParallel(3);
var boundedResult = boundedParallel.For(0, 0, (n) => { });
Assert.AreEqual(1, boundedParallel.Stats.TotalSerialRunCount);
Assert.AreEqual(0, boundedParallel.Stats.TotalParallelRunCount);
Assert.AreEqual(regularParallelResult.IsCompleted, boundedResult.IsCompleted);
}
public void TestBoundedParallelForForceSerial()
{
var items = new[] { 1, 2, 3 };
var sumItems = 0;
// ReSharper disable once RedundantArgumentDefaultValue
var boundedParallel = new BoundedParallel(2);
Parallel.Invoke(() =>
{
boundedParallel.For(0, 3, (idx) =>
{
Thread.Sleep(500);
Interlocked.Add(ref sumItems, items[idx]);
});
}, () =>
{
boundedParallel.For(0, 3, (idx) =>
{
Thread.Sleep(500);
Interlocked.Add(ref sumItems, items[idx]);
});
}, () =>
{
boundedParallel.For(0, 3, (idx) =>
{
Thread.Sleep(500);
Interlocked.Add(ref sumItems, items[idx]);
});
}, () =>
{
boundedParallel.For(0, 3, (idx) =>
{
Thread.Sleep(500);
Interlocked.Add(ref sumItems, items[idx]);
});
});
Assert.AreEqual(24, sumItems);
Assert.IsTrue(boundedParallel.Stats.TotalSerialRunCount > 0, "TotalSerialRunCount should be > 0");
}
public virtual void Replicate(int readBufferSize, int insertBatchSize)
{
if (!_prepared)
{
throw new InvalidOperationException("DataReplicator not prepared. Can't run.");
}
try
{
_parallelRunner.For(0, _readers.Count, i =>
{
EnsureTableCreated(i);
_readers[i] ??= _sourceCmds[i].ExecuteReader();
var sourceAdapter = GenericObjectBuilder.Build <TSourceAdapter>(_readers[i], readBufferSize);
sourceAdapter.AbortOnReadException = true;
var columnNames = new List <string>();
foreach (var meta in ColumnMetadataLists[i])
{
columnNames.Add(meta.ColumnName);
}
var targetAdapter = GenericObjectBuilder.Build <TTargetAdapter>(_sourceTables[i].Item1, columnNames, _targetConnections[i], insertBatchSize);
targetAdapter.UseNativeTypeConvertor = UseNativeTypeConvertor;
targetAdapter.AbortOnProcessException = true;
targetAdapter.LiteralParamBinding = LiteralParamBinding;
var pipeline = new TDataPipeline();
ConfigureTargetConnection(_targetConnections[i], columnNames.Count, insertBatchSize);
var tran = UseTransaction ? _targetConnections[i].BeginTransaction() : null;
try
{
if (tran != null)
{
targetAdapter.BeforeCommandPrepare += (adapter, cmd) => cmd.Transaction = tran;
}
pipeline.Pump(sourceAdapter, targetAdapter);
tran?.Commit();
}
catch (Exception)
{
tran?.Rollback();
throw;
}
});
}
finally
{
Disposer.Dispose(_readers);
}
}
public void TestBoundedParallelSimpleForCall()
{
var items = new[] { 1, 2, 3 };
var sumItems = 0;
var boundedParallel = new BoundedParallel(3);
boundedParallel.For(0, 3, (idx) =>
{
Interlocked.Add(ref sumItems, items[idx]);
});
Assert.AreEqual(6, sumItems);
Assert.IsTrue(boundedParallel.Stats.TotalSerialRunCount == 0, "TotalSerialRunCount must be zero");
}
public void Pump(IList <ISourceAdapter <TRow> > sourceAdapters, RowsJoinedDelegate onRowsJoined)
{
try
{
_finishThread = false;
_itemAvailableEvent = new ManualResetEventSlim(false);
var targetAdapters = new List <PassThruTargetAdapter <TRow> >();
foreach (var dummy in sourceAdapters)
{
targetAdapters.Add(new PassThruTargetAdapter <TRow>(ProcessRow)
{
AbortOnProcessException = true
});
}
_rowsQueues = new ConcurrentQueue <TRow> [targetAdapters.Count];
_rowsProcessedCount = new int[targetAdapters.Count];
_finished = new bool[targetAdapters.Count];
for (var i = 0; i < _rowsQueues.Length; i++)
{
_rowsQueues[i] = new ConcurrentQueue <TRow>();
}
var processingThread = new Thread(ProcessRows);
processingThread.Start(onRowsJoined);
var boundedParallel = new BoundedParallel(1, targetAdapters.Count);
boundedParallel.For(0, sourceAdapters.Count, index =>
{
var pipeline = new DataPipeline <TRow>();
targetAdapters[index].Id = index;
pipeline.AfterTargetAdapterProcessRow += (a, row) =>
{
_rowsProcessedCount[index]++;
for (var idx = 0; idx < _finished.Length; idx++)
{
if (_finished[idx] && _rowsProcessedCount[index] > _rowsProcessedCount[idx])
{
throw new DataPipelineAbortedException();
}
}
};
pipeline.Pump(sourceAdapters[index], targetAdapters[index]);
_finished[index] = true;
});
_finishThread = true;
_itemAvailableEvent.Set();
processingThread.Join();
if (_threadException != null)
{
throw _threadException;
}
CheckOutOfBalanceSourceAdapters();
foreach (var queue in _rowsQueues)
{
if (!queue.IsEmpty)
{
throw new DataPipelineAbortedException();
}
}
}
finally
{
_threadException = null;
_rowsQueues = null;
_itemAvailableEvent = null;
}
}
public UnitTestConcurrentStackedBagSlim()
{
_parallel = new BoundedParallel(2, 4);
_parallel.For(0, 1000, i => { }); // warm-up
}
