private Result <IssueData[]> ProcessIssueBatch(ElasticThreadPool pool, string projectId, string filterId, int pageNumber, int issuesInBatch, Tuplet <bool> canceled, Result <IssueData[]> result)
{
pool.QueueWorkItem(HandlerUtil.WithEnv(delegate {
// TODO (steveb): use result.IsCanceled instead of shared tuple once cancellation is supported on the result object
// check if request has been canceled
if (!canceled.Item1)
{
IssueData[] issuesForBatch;
if (!string.IsNullOrEmpty(filterId))
{
issuesForBatch = _service.mc_filter_get_issues(_username, _password, projectId, filterId, pageNumber.ToString(), issuesInBatch.ToString());
}
else
{
issuesForBatch = _service.mc_project_get_issues(_username, _password, projectId, pageNumber.ToString(), issuesInBatch.ToString());
}
result.Return(issuesForBatch);
}
else
{
// TODO (steveb): throw a more specific exception
result.Throw(new Exception("unspecified error"));
}
}, TaskEnv.Clone()));
return(result);
}
public void New_TryEnqueueItem_x50_TryEnqueueConsumer_x50_ItemCount_ConsumerCount()
{
var q = new LockFreeItemConsumerQueue <int>();
var etp = new ElasticThreadPool(10, 10);
// submit enqueue & dequeue work-items
const int max = 10000;
int count = max + 1;
var e = new ManualResetEvent(false);
int[] checks = new int[max];
for (int i = 0; i < max; ++i)
{
int j = i;
etp.TryQueueWorkItem(() => {
int k = Interlocked.Increment(ref checks[j]);
Assert.AreEqual(1, k, "value for {0} was already increased", j);
q.TryEnqueue(j);
});
etp.TryQueueWorkItem(() => {
q.TryEnqueue(x => {
int k = Interlocked.Decrement(ref checks[x]);
Assert.AreEqual(0, k, "value for {0} was already decreased", x);
// ReSharper disable AccessToModifiedClosure
if (Interlocked.Decrement(ref count) == 0)
{
// ReSharper restore AccessToModifiedClosure
e.Set();
}
});
});
}
if (Interlocked.Decrement(ref count) == 0)
{
e.Set();
}
if (!e.WaitOne(TimeSpan.FromSeconds(10)))
{
Assert.Fail("test timed out");
}
for (int i = 0; i < max; ++i)
{
Assert.AreEqual(0, checks[i], "entry {0}", i);
}
Assert.AreEqual(0, q.ItemCount);
Assert.IsTrue(q.ItemIsEmpty);
Assert.AreEqual(0, q.ConsumerCount);
Assert.IsTrue(q.ConsumerIsEmpty);
}
public void ElasticThreadPool_Fibonacci_Min_0_Max_4()
{
var stp = new ElasticThreadPool(0, 4);
int value;
TimeSpan elapsed;
FibonacciThreadPool(stp, 30, TimeSpan.Zero, out value, out elapsed);
Assert.AreEqual(832040, value);
_log.Debug("Time: " + elapsed);
_log.Debug("Work items processed: " + _counter);
stp.Dispose();
Assert.AreEqual(0, stp.WorkItemCount, "WorkQueue items");
Assert.AreEqual(0, stp.ThreadCount, "WorkQueue threads");
}
public void ElasticThreadPool_Fibonacci_Min_0_Max_100_with_1ms_delay()
{
var stp = new ElasticThreadPool(0, 100);
int value;
TimeSpan elapsed;
FibonacciThreadPool(stp, 25, TimeSpan.FromSeconds(0.001), out value, out elapsed);
Assert.AreEqual(75025, value);
_log.Debug("Time: " + elapsed);
_log.Debug("Work items processed: " + _counter);
stp.Dispose();
Assert.AreEqual(0, stp.WorkItemCount, "WorkQueue items");
Assert.AreEqual(0, stp.ThreadCount, "WorkQueue threads");
}
//--- Constructors ---
static Async()
{
if (!int.TryParse(System.Configuration.ConfigurationManager.AppSettings["threadpool-min"], out _minThreads))
{
_minThreads = 4;
}
if (!int.TryParse(System.Configuration.ConfigurationManager.AppSettings["threadpool-max"], out _maxThreads))
{
_maxThreads = 200;
}
int maxStackSize;
if (int.TryParse(System.Configuration.ConfigurationManager.AppSettings["max-stacksize"], out maxStackSize))
{
_maxStackSize = maxStackSize;
}
// check which global dispatch queue implementation to use
int dummy;
switch (System.Configuration.ConfigurationManager.AppSettings["threadpool"])
{
default:
case "elastic":
ThreadPool.GetMinThreads(out dummy, out _minPorts);
ThreadPool.GetMaxThreads(out dummy, out _maxPorts);
_log.DebugFormat("Using ElasticThreadPool with {0}min / {1}max", _minThreads, _maxThreads);
var elasticThreadPool = new ElasticThreadPool(_minThreads, _maxThreads);
GlobalDispatchQueue = elasticThreadPool;
_inplaceActivation = false;
_availableThreadsCallback = delegate(out int threads, out int ports) {
int dummy2;
ThreadPool.GetAvailableThreads(out dummy2, out ports);
threads = elasticThreadPool.MaxParallelThreads - elasticThreadPool.ThreadCount;
};
break;
case "legacy":
ThreadPool.GetMinThreads(out dummy, out _minPorts);
ThreadPool.GetMaxThreads(out dummy, out _maxPorts);
ThreadPool.SetMinThreads(_minThreads, _minPorts);
ThreadPool.SetMaxThreads(_maxThreads, _maxPorts);
_log.Debug("Using LegacyThreadPool");
GlobalDispatchQueue = LegacyThreadPool.Instance;
_availableThreadsCallback = ThreadPool.GetAvailableThreads;
break;
}
}
public void ElasticThreadPool_Fibonacci_from_1_to_33_threads()
{
var throughputs = new TimeSpan[33];
for (int i = 1; i < throughputs.Length; ++i)
{
using (var stp = new ElasticThreadPool(i, i)) {
int value;
TimeSpan elapsed;
FibonacciThreadPool(stp, 30, TimeSpan.Zero, out value, out elapsed);
Assert.AreEqual(832040, value);
throughputs[i] = elapsed;
}
}
_log.Debug("--- Results ---");
for (int i = 1; i < throughputs.Length; ++i)
{
_log.DebugFormat("{0,2}: {1}", i, throughputs[i]);
}
}
public void ElasticThreadPool_Multi_Staged_Fibonacci_Min_1_Max_30()
{
const int test = 4;
// initialize data structures
ElasticThreadPool[] stp = new ElasticThreadPool[test];
Result <int>[] results = new Result <int> [test];
for (int i = 0; i < test; ++i)
{
stp[i] = new ElasticThreadPool(1, 30);
results[i] = new Result <int>(TimeSpan.MaxValue, TaskEnv.New(stp[i]));
}
// start test
var sw = Stopwatch.StartNew();
for (int i = 0; i < results.Length; ++i)
{
_log.DebugFormat("--- FIBONACCI KICK-OFF: {0}", i);
Fibonacci(stp[i], 30, TimeSpan.Zero, results[i]);
Thread.Sleep(TimeSpan.FromSeconds(1));
}
results.Join(new Result(TimeSpan.MaxValue)).Wait();
sw.Stop();
TimeSpan elapsed = sw.Elapsed;
// check results
for (int i = 0; i < test; ++i)
{
Assert.AreEqual(832040, results[i].Value, "result {0} did not match", i);
}
_log.Debug("Time: " + elapsed);
_log.Debug("Work items processed: " + _counter);
for (int i = 0; i < test; ++i)
{
stp[i].Dispose();
Assert.AreEqual(0, stp[i].WorkItemCount, "WorkQueue[{0}] items", i);
Assert.AreEqual(0, stp[i].ThreadCount, "WorkQueue[{0}] threads", i);
}
}
private IssueData[] RetrieveIssueData(string username, string password, string projectId, string filterId, int pageNumber, int numberPerPage)
{
using (ElasticThreadPool pool = new ElasticThreadPool(0, 2)) {
List <IssueData> result = new List <IssueData>();
List <Result <IssueData[]> > results = new List <Result <IssueData[]> >();
Tuplet <bool> canceled = new Tuplet <bool>(false);
for (int issuesRemaining = numberPerPage; issuesRemaining > 0; issuesRemaining -= MAX_ISSUES_IN_REQUEST, ++pageNumber)
{
int issuesInBatch = Math.Min(issuesRemaining, MAX_ISSUES_IN_REQUEST);
results.Add(ProcessIssueBatch(pool, projectId, filterId, pageNumber, issuesInBatch, canceled, new Result <IssueData[]>(TimeSpan.FromSeconds(30))));
}
Dictionary <string, IssueData> tempHash = new Dictionary <string, IssueData>();
foreach (Result <IssueData[]> r in results)
{
IssueData[] batch = r.Wait();
//HACK: Workaround for Mantis's broken paging: Asking for a batch at a page number that doesnt exist
// will return the first page's results.
// This takes care of the case when the #of tickets is evenly divisible by the batch size. (i.e 100 tix, 20/page)
foreach (IssueData bug in batch)
{
if (!tempHash.ContainsKey(bug.id))
{
tempHash[bug.id] = bug;
result.Add(bug);
}
}
if (batch.Length < MAX_ISSUES_IN_REQUEST)
{
//the current batch did not fill up, don't go to the next batch
canceled.Item1 = true;
break;
}
}
return(result.ToArray());
}
}
