public void ForEachTest()
{
Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF));
ExceptionCollector ec = new ExceptionCollector();
AsyncQueue <int> q1 = new AsyncQueue <int>(5);
Func <Task> t1 = async delegate()
{
for (int i = 0; i < 100; ++i)
{
await q1.Enqueue(r.Next(1000), ec.CancellationToken);
}
q1.WriteEof();
};
AsyncQueue <int> q2 = new AsyncQueue <int>(5);
Func <Task> t2 = WorkerTask.ForEach
(
q1,
async delegate(ForEachInfo <int> fi)
{
await q2.Enqueue(fi.Item * 100, fi.CancellationToken);
},
delegate()
{
q2.WriteEof();
return(Task.CompletedTask);
},
ec
);
List <int> items = new List <int>();
Func <Task> t3 = WorkerTask.ForEach
(
q2,
delegate(ForEachInfo <int> fi)
{
items.Add(fi.Item);
return(Task.CompletedTask);
},
delegate()
{
return(Task.CompletedTask);
},
ec
);
Task T1 = Task.Run(t1);
Task T2 = Task.Run(t2);
Task T3 = Task.Run(t3);
Task.WaitAll(T1, T2, T3);
Assert.AreEqual(0, ec.Exceptions.Count);
Assert.AreEqual(100, items.Count);
}
public static IQueueSource <U> Select <T, U>(this IQueueSource <T> queue, Func <T, Task <U> > func, int?capacity = null)
{
AsyncQueue <U> outQueue = new AsyncQueue <U>(capacity ?? 2);
Func <Task> worker = async delegate()
{
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (!item.HasValue)
{
break;
}
U item2 = await func(item.Value);
await outQueue.Enqueue(item2, CancellationToken.None);
}
outQueue.WriteEof();
};
Task.Run(worker);
return(outQueue);
}
public static IQueueSource <T> Where <T>(this IQueueSource <T> queue, Func <T, Task <bool> > predicate, int?capacity = null)
{
AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2);
Func <Task> worker = async delegate()
{
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (!item.HasValue)
{
break;
}
if (await predicate(item.Value))
{
await outQueue.Enqueue(item.Value, CancellationToken.None);
}
}
outQueue.WriteEof();
};
Task _dummy = Task.Run(worker);
return(outQueue);
}
private async Task PutItems(string prefix, AsyncQueue <string> dest, List <Item> items)
{
foreach (Item i in items)
{
await Task.Delay(i.delay);
await dest.Enqueue(prefix + i.value, CancellationToken.None);
}
dest.WriteEof();
}
private async Task ExtMethodQueueTestAsync()
{
AsyncQueue <string> q = new AsyncQueue <string>(5);
Func <Task> producer = async delegate()
{
#region
for (int i = 0; i < 20; ++i)
{
System.Diagnostics.Debug.WriteLine($"Writing {i}...");
await q.Enqueue($"Value: {i}", CancellationToken.None);
}
System.Diagnostics.Debug.WriteLine("Writing EOF...");
q.WriteEof();
#endregion
};
Func <Task> consumer = async delegate()
{
#region
bool more = true;
while (more)
{
System.Diagnostics.Debug.WriteLine("Reading...");
Option <string> ostr = await q.Dequeue(CancellationToken.None);
if (ostr.HasValue)
{
System.Diagnostics.Debug.WriteLine($"Read {ostr.Value}");
}
else
{
System.Diagnostics.Debug.WriteLine("Read EOF...");
more = false;
}
}
#endregion
};
Task tProducer = Task.Run(producer);
Task tConsumer = Task.Run(consumer);
await Task.WhenAll(tProducer, tConsumer);
}
public static IQueueSource <T> ParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null)
{
AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2);
IdleDetector idleDetector = new IdleDetector();
Func <Task> workPoster = async delegate()
{
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (!item.HasValue)
{
break;
}
T itemValue = item.Value;
idleDetector.Enter();
Task doingWork = await parallelWorker.StartWorkItem
(
async (int workerId) =>
{
try
{
if (await predicate(itemValue))
{
await outQueue.Enqueue(itemValue, CancellationToken.None);
}
}
finally
{
idleDetector.Leave();
}
},
CancellationToken.None
);
}
await idleDetector.WaitForIdle(CancellationToken.None);
outQueue.WriteEof();
};
Task.Run(workPoster);
return(outQueue);
}
public static IQueueSource <T> AsQueueSource <T>(this IEnumerable <T> items, int?capacity = null)
{
AsyncQueue <T> queue = new AsyncQueue <T>(capacity ?? 2);
Func <Task> feed = async delegate()
{
foreach (var item in items)
{
await queue.Enqueue(item, CancellationToken.None);
}
queue.WriteEof();
};
Task _dummy = Task.Run(feed);
return(queue);
}
public static IQueueSource <T> Reorder <T>(this IQueueSource <T> queue, Func <T, int> getOrder, int first, int?capacity = null)
{
AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2);
Func <Task> worker = async delegate()
{
int next = first;
ImmutableDictionary <int, T> buffer = ImmutableDictionary <int, T> .Empty;
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (!item.HasValue)
{
break;
}
int order = getOrder(item.Value);
if (order == next)
{
await outQueue.Enqueue(item.Value, CancellationToken.None);
++next;
while (buffer.ContainsKey(next))
{
await outQueue.Enqueue(buffer[next], CancellationToken.None);
buffer = buffer.Remove(next);
++next;
}
}
else
{
buffer = buffer.Add(order, item.Value);
}
}
outQueue.WriteEof();
};
Task.Run(worker);
return(outQueue);
}
public void MultiSourceForEachExceptionTest()
{
ExceptionCollector ec = new ExceptionCollector();
AsyncQueue <int> q1 = new AsyncQueue <int>(5);
Func <Task> t1 = async delegate()
{
for (int i = 0; i < 100; ++i)
{
await q1.Enqueue(i, ec.CancellationToken);
}
q1.WriteEof();
};
AsyncQueue <int> q2 = new AsyncQueue <int>(5);
Func <Task> t2 = async delegate()
{
for (int i = 100; i < 200; ++i)
{
await q2.Enqueue(i, ec.CancellationToken);
}
q2.WriteEof();
};
AsyncQueue <int> q3 = new AsyncQueue <int>(5);
Func <Task> t3 = WorkerTask.ForEach
(
new IQueueSource <int>[] { q1, q2 },
InputPriorities.RoundRobin,
async fei =>
{
if (fei.Item == 133)
{
throw new InvalidOperationException("test");
}
await q3.Enqueue(fei.Item, fei.CancellationToken);
},
() =>
{
q3.WriteEof();
return(Task.CompletedTask);
},
ec
);
Func <Task> t4 = WorkerTask.ForEach
(
q3,
fei =>
{
System.Diagnostics.Debug.WriteLine(fei.Item);
return(Task.CompletedTask);
},
null,
ec
);
Task T1 = Task.Run(t1);
Task T2 = Task.Run(t2);
Task T3 = Task.Run(t3);
Task T4 = Task.Run(t4);
try
{
ec.WaitAll(T1, T2, T3, T4);
}
catch (Exception exc)
{
System.Diagnostics.Debug.WriteLine(exc);
}
}
public void ParallelForEachExceptionTest()
{
ExceptionCollector ec = new ExceptionCollector();
AsyncQueue <int> q1 = new AsyncQueue <int>(5);
Func <Task> t1 = async delegate()
{
for (int i = 0; i < 100; ++i)
{
await q1.Enqueue(i, ec.CancellationToken);
}
q1.WriteEof();
};
AsyncQueue <int> q2 = new AsyncQueue <int>(5);
Func <Task> t2 = WorkerTask.ForEach
(
q1,
async delegate(ForEachInfo <int> fi)
{
await q2.Enqueue(fi.Item * 100, fi.CancellationToken);
if (fi.Item == 53)
{
throw new InvalidOperationException("test 1");
}
},
delegate()
{
q2.WriteEof();
return(Task.CompletedTask);
},
ec
);
List <int> items = new List <int>();
Func <Task> t3 = WorkerTask.ForEach
(
q2,
delegate(ForEachInfo <int> fi)
{
items.Add(fi.Item);
return(Task.CompletedTask);
},
delegate()
{
return(Task.CompletedTask);
},
ec
);
Task T1 = Task.Run(t1);
Task T2 = Task.Run(t2);
Task T3 = Task.Run(t3);
bool threwException = false;
try
{
ec.WaitAll(T1, T2, T3);
}
catch (Exception exc)
{
System.Diagnostics.Debug.WriteLine(exc);
threwException = true;
}
Assert.IsTrue(threwException);
}
private async Task CompleteAnyWithCancellationAsync()
{
AsyncQueue <string> q1 = new AsyncQueue <string>(5);
AsyncQueue <string> q2 = new AsyncQueue <string>(5);
Func <Task> producer1 = async delegate()
{
await Task.Delay(2500);
await q1.Enqueue("one", CancellationToken.None);
q1.WriteEof();
};
Func <Task> producer2 = async delegate()
{
await Task.Delay(5500);
await q2.Enqueue("two", CancellationToken.None);
q2.WriteEof();
};
Func <Task> consumer = async delegate()
{
bool q1eof = false;
bool q2eof = false;
while (!q1eof || !q2eof)
{
Tuple <int, string> r1 = null;
using (CancellationTokenSource s1 = new CancellationTokenSource(1000))
{
try
{
r1 = await Utils.OperationStarters <int, string>()
.AddIf(!q1eof, 1, Utils.StartableGet(q1, x => x, null))
.AddIf(!q2eof, 2, Utils.StartableGet(q2, x => x, null))
.CompleteAny(s1.Token);
}
catch (OperationCanceledException)
{
}
}
if (r1 == null)
{
System.Diagnostics.Debug.WriteLine("Cancelled (because of timeout)");
}
else if (r1.Item1 == 1)
{
if (r1.Item2 == null)
{
System.Diagnostics.Debug.WriteLine("Got EOF from 1");
q1eof = true;
}
else
{
System.Diagnostics.Debug.WriteLine($"Got {r1.Item2} from 1");
}
}
else
{
Assert.AreEqual(2, r1.Item1);
if (r1.Item2 == null)
{
System.Diagnostics.Debug.WriteLine("Got EOF from 2");
q2eof = true;
}
else
{
System.Diagnostics.Debug.WriteLine($"Got {r1.Item2} from 2");
}
}
}
};
Task p1 = Task.Run(producer1);
Task p2 = Task.Run(producer2);
Task c = Task.Run(consumer);
await Task.WhenAll(p1, p2, c);
}
private async Task CompleteAnyWithCancellationAsync()
{
AsyncQueue<string> q1 = new AsyncQueue<string>(5);
AsyncQueue<string> q2 = new AsyncQueue<string>(5);
Func<Task> producer1 = async delegate ()
{
await Task.Delay(2500);
await q1.Enqueue("one", CancellationToken.None);
q1.WriteEof();
};
Func<Task> producer2 = async delegate ()
{
await Task.Delay(5500);
await q2.Enqueue("two", CancellationToken.None);
q2.WriteEof();
};
Func<Task> consumer = async delegate ()
{
bool q1eof = false;
bool q2eof = false;
while (!q1eof || !q2eof)
{
Tuple<int, string> r1 = null;
using (CancellationTokenSource s1 = new CancellationTokenSource(1000))
{
try
{
r1 = await Utils.OperationStarters<int, string>()
.AddIf(!q1eof, 1, Utils.StartableGet(q1, x => x, null))
.AddIf(!q2eof, 2, Utils.StartableGet(q2, x => x, null))
.CompleteAny(s1.Token);
}
catch (OperationCanceledException)
{
}
}
if (r1 == null)
{
System.Diagnostics.Debug.WriteLine("Cancelled (because of timeout)");
}
else if (r1.Item1 == 1)
{
if (r1.Item2 == null)
{
System.Diagnostics.Debug.WriteLine("Got EOF from 1");
q1eof = true;
}
else
{
System.Diagnostics.Debug.WriteLine($"Got {r1.Item2} from 1");
}
}
else
{
Assert.AreEqual(2, r1.Item1);
if (r1.Item2 == null)
{
System.Diagnostics.Debug.WriteLine("Got EOF from 2");
q2eof = true;
}
else
{
System.Diagnostics.Debug.WriteLine($"Got {r1.Item2} from 2");
}
}
}
};
Task p1 = Task.Run(producer1);
Task p2 = Task.Run(producer2);
Task c = Task.Run(consumer);
await Task.WhenAll(p1, p2, c);
}
private async Task PutItems(string prefix, AsyncQueue<string> dest, List<Item> items)
{
foreach(Item i in items)
{
await Task.Delay(i.delay);
await dest.Enqueue(prefix + i.value, CancellationToken.None);
}
dest.WriteEof();
}
private async Task ExtMethodQueueTestAsync()
{
AsyncQueue<string> q = new AsyncQueue<string>(5);
Func<Task> producer = async delegate ()
{
#region
for (int i = 0; i < 20; ++i)
{
System.Diagnostics.Debug.WriteLine($"Writing {i}...");
await q.Enqueue($"Value: {i}", CancellationToken.None);
}
System.Diagnostics.Debug.WriteLine("Writing EOF...");
q.WriteEof();
#endregion
};
Func<Task> consumer = async delegate ()
{
#region
bool more = true;
while (more)
{
System.Diagnostics.Debug.WriteLine("Reading...");
Option<string> ostr = await q.Dequeue(CancellationToken.None);
if (ostr.HasValue)
{
System.Diagnostics.Debug.WriteLine($"Read {ostr.Value}");
}
else
{
System.Diagnostics.Debug.WriteLine("Read EOF...");
more = false;
}
}
#endregion
};
Task tProducer = Task.Run(producer);
Task tConsumer = Task.Run(consumer);
await Task.WhenAll(tProducer, tConsumer);
}
private async Task SimpleQueueTestAsync()
{
AsyncQueue<int> queue = new AsyncQueue<int>(11);
Func<Task> producer = async delegate()
{
#region
for (int i = 0; i < 20; ++i)
{
System.Diagnostics.Debug.WriteLine("Acquiring write...");
AcquireWriteResult result = await queue.AcquireWriteAsync(1, CancellationToken.None);
result.Visit<DBNull>
(
new Func<AcquireWriteSucceeded, DBNull>
(
succeeded =>
{
System.Diagnostics.Debug.WriteLine("Write-acquire succeeded, offset " + succeeded.Offset + ", acquired " + succeeded.ItemCount + " spaces");
Assert.AreEqual(1, succeeded.ItemCount);
if (succeeded.ItemCount >= 1)
{
System.Diagnostics.Debug.WriteLine("Releasing write (1)...");
queue.ReleaseWrite(i);
}
else
{
System.Diagnostics.Debug.WriteLine("Releasing write (0)...");
queue.ReleaseWrite();
}
return DBNull.Value;
}
),
new Func<AcquireWriteCancelled, DBNull>
(
cancelled =>
{
throw new OperationCanceledException();
}
),
new Func<AcquireWriteFaulted, DBNull>
(
faulted =>
{
throw faulted.Exception;
}
)
);
}
System.Diagnostics.Debug.WriteLine("Writing EOF...");
queue.WriteEof();
#endregion
};
Func<Task> consumer = async delegate()
{
#region
bool more = true;
while (more)
{
System.Diagnostics.Debug.WriteLine("Acquiring read...");
const int ACQUIRE_COUNT = 3;
AcquireReadResult result = await queue.AcquireReadAsync(ACQUIRE_COUNT, CancellationToken.None);
result.Visit<DBNull>
(
new Func<AcquireReadSucceeded, DBNull>
(
succeeded =>
{
System.Diagnostics.Debug.WriteLine("Read-acquire succeeded, offset " + succeeded.Offset + ", acquired " + succeeded.ItemCount + " items");
Assert.IsInstanceOfType(succeeded, typeof(AcquireReadSucceeded<int>));
if (succeeded is AcquireReadSucceeded<int>)
{
AcquireReadSucceeded<int> succeeded2 = (AcquireReadSucceeded<int>)succeeded;
System.Diagnostics.Debug.WriteLine("{ " + string.Join(", ", succeeded2.Items) + " }");
}
if (succeeded.ItemCount < ACQUIRE_COUNT)
{
System.Diagnostics.Debug.WriteLine("Setting \"more\" flag to false...");
more = false;
}
System.Diagnostics.Debug.WriteLine("Releasing read (" + succeeded.ItemCount + ")...");
queue.ReleaseRead(succeeded.ItemCount);
return DBNull.Value;
}
),
new Func<AcquireReadCancelled, DBNull>
(
cancelled =>
{
throw new OperationCanceledException();
}
),
new Func<AcquireReadFaulted, DBNull>
(
faulted =>
{
throw faulted.Exception;
}
)
);
}
#endregion
};
Task tProducer = Task.Run(producer);
Task tConsumer = Task.Run(consumer);
await Task.WhenAll(tProducer, tConsumer);
}
public void ParallelMultiSourceForEachExceptionTest()
{
object rSyncRoot = new object();
Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF));
ExceptionCollector ec = new ExceptionCollector();
AsyncQueue <int> q1 = new AsyncQueue <int>(5);
Func <Task> t1 = async delegate()
{
for (int i = 0; i < 100; ++i)
{
await q1.Enqueue(i, ec.CancellationToken);
}
q1.WriteEof();
};
AsyncQueue <int> q2 = new AsyncQueue <int>(5);
Func <Task> t2 = async delegate()
{
for (int i = 100; i < 200; ++i)
{
await q2.Enqueue(i, ec.CancellationToken);
}
q2.WriteEof();
};
AsyncQueue <int> q3 = new AsyncQueue <int>(5);
ParallelWorker pw = new ParallelWorker(8);
Func <Task> t3 = WorkerTask.ParallelForEach
(
new IQueueSource <int>[] { q1, q2 },
InputPriorities.RoundRobin,
pw,
async fei =>
{
await q3.Enqueue(fei.Item, fei.CancellationToken);
if (fei.Item == 133)
{
throw new InvalidOperationException("test");
}
int time;
lock (rSyncRoot)
{
time = 100 + r.Next(800);
}
await Task.Delay(time);
},
() =>
{
q3.WriteEof();
return(Task.CompletedTask);
},
ec
);
Func <Task> t4 = WorkerTask.ForEach
(
q3,
fei =>
{
System.Diagnostics.Debug.WriteLine(fei.Item);
return(Task.CompletedTask);
},
null,
ec
);
Task T1 = Task.Run(t1);
Task T2 = Task.Run(t2);
Task T3 = Task.Run(t3);
Task T4 = Task.Run(t4);
try
{
ec.WaitAll(T1, T2, T3, T4);
}
catch (Exception exc)
{
System.Diagnostics.Debug.WriteLine(exc);
}
}
public void ForEachWithExceptionTest()
{
Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF));
ExceptionCollector ec = new ExceptionCollector();
AsyncQueue <int> q1 = new AsyncQueue <int>(5);
Func <Task> t1 = async delegate()
{
for (int i = 0; i < 100; ++i)
{
await q1.Enqueue(r.Next(1000), ec.CancellationToken);
}
q1.WriteEof();
};
AsyncQueue <int> q2 = new AsyncQueue <int>(5);
Func <Task> t2 = WorkerTask.ForEach
(
q1,
async delegate(ForEachInfo <int> fi)
{
await q2.Enqueue(fi.Item * 100, fi.CancellationToken);
throw new InvalidOperationException("test 1");
},
delegate()
{
q2.WriteEof();
return(Task.FromException(new InvalidOperationException("test 2")));
},
ec
);
List <int> items = new List <int>();
Func <Task> t3 = WorkerTask.ForEach
(
q2,
delegate(ForEachInfo <int> fi)
{
items.Add(fi.Item);
return(Task.CompletedTask);
},
delegate()
{
return(Task.CompletedTask);
},
ec
);
Task T1 = Task.Run(t1);
Task T2 = Task.Run(t2);
Task T3 = Task.Run(t3);
bool threwException = false;
try
{
ec.WaitAll(T1, T2, T3);
}
catch (Exception exc)
{
System.Diagnostics.Debug.WriteLine(exc);
threwException = true;
}
Assert.IsTrue(threwException);
}
private async Task SimpleQueueTestAsync()
{
AsyncQueue <int> queue = new AsyncQueue <int>(11);
Func <Task> producer = async delegate()
{
#region
for (int i = 0; i < 20; ++i)
{
System.Diagnostics.Debug.WriteLine("Acquiring write...");
AcquireWriteResult result = await queue.AcquireWriteAsync(1, CancellationToken.None);
result.Visit <DBNull>
(
new Func <AcquireWriteSucceeded, DBNull>
(
succeeded =>
{
System.Diagnostics.Debug.WriteLine("Write-acquire succeeded, offset " + succeeded.Offset + ", acquired " + succeeded.ItemCount + " spaces");
Assert.AreEqual(1, succeeded.ItemCount);
if (succeeded.ItemCount >= 1)
{
System.Diagnostics.Debug.WriteLine("Releasing write (1)...");
queue.ReleaseWrite(i);
}
else
{
System.Diagnostics.Debug.WriteLine("Releasing write (0)...");
queue.ReleaseWrite();
}
return(DBNull.Value);
}
),
new Func <AcquireWriteCancelled, DBNull>
(
cancelled =>
{
throw new OperationCanceledException();
}
),
new Func <AcquireWriteFaulted, DBNull>
(
faulted =>
{
throw faulted.Exception;
}
)
);
}
System.Diagnostics.Debug.WriteLine("Writing EOF...");
queue.WriteEof();
#endregion
};
Func <Task> consumer = async delegate()
{
#region
bool more = true;
while (more)
{
System.Diagnostics.Debug.WriteLine("Acquiring read...");
const int ACQUIRE_COUNT = 3;
AcquireReadResult result = await queue.AcquireReadAsync(ACQUIRE_COUNT, CancellationToken.None);
result.Visit <DBNull>
(
new Func <AcquireReadSucceeded, DBNull>
(
succeeded =>
{
System.Diagnostics.Debug.WriteLine("Read-acquire succeeded, offset " + succeeded.Offset + ", acquired " + succeeded.ItemCount + " items");
Assert.IsInstanceOfType(succeeded, typeof(AcquireReadSucceeded <int>));
if (succeeded is AcquireReadSucceeded <int> )
{
AcquireReadSucceeded <int> succeeded2 = (AcquireReadSucceeded <int>)succeeded;
System.Diagnostics.Debug.WriteLine("{ " + string.Join(", ", succeeded2.Items) + " }");
}
if (succeeded.ItemCount < ACQUIRE_COUNT)
{
System.Diagnostics.Debug.WriteLine("Setting \"more\" flag to false...");
more = false;
}
System.Diagnostics.Debug.WriteLine("Releasing read (" + succeeded.ItemCount + ")...");
queue.ReleaseRead(succeeded.ItemCount);
return(DBNull.Value);
}
),
new Func <AcquireReadCancelled, DBNull>
(
cancelled =>
{
throw new OperationCanceledException();
}
),
new Func <AcquireReadFaulted, DBNull>
(
faulted =>
{
throw faulted.Exception;
}
)
);
}
#endregion
};
Task tProducer = Task.Run(producer);
Task tConsumer = Task.Run(consumer);
await Task.WhenAll(tProducer, tConsumer);
}