public static IQueueSource <U> Select <T, U>(this IQueueSource <T> queue, Func <T, Task <U> > func, int?capacity = null)
{
AsyncQueue <U> outQueue = new AsyncQueue <U>(capacity ?? 2);
Func <Task> worker = async delegate {
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (!item.HasValue)
{
break;
}
U item2 = await func(item.Value);
await outQueue.Enqueue(item2, CancellationToken.None);
}
outQueue.WriteEof();
};
Task.Run(worker);
return(outQueue);
}
public static Func <Task> ParallelForEach <T>(IQueueSource <T> source, ParallelWorker parallelWorker, Func <ForEachInfo <T>, Task> processAsync, Func <Task> onCloseAsync, ExceptionCollector ec)
{
Func <Task> t = async delegate()
{
IdleDetector idleDetector = new IdleDetector();
try
{
while (true)
{
var item = await source.Dequeue(ec.CancellationToken);
if (!item.HasValue)
{
break;
}
T itemValue = item.Value;
idleDetector.Enter();
await parallelWorker.StartWorkItem
(
async workerId =>
{
try
{
await processAsync(new ForEachInfo <T>(itemValue, 0, workerId, ec.CancellationToken));
}
catch (Exception exc)
{
ec.Add(exc);
}
finally
{
idleDetector.Leave();
}
},
ec.CancellationToken
);
}
}
finally
{
await idleDetector.WaitForIdle(CancellationToken.None);
if (onCloseAsync != null)
{
try
{
await onCloseAsync();
}
catch (Exception exc)
{
ec.Add(exc);
}
}
}
};
return(t);
}
public void FindResourcesByType_Execute_ExpectIQueueSource()
{
var expected = new IQueueSource[]
{
new RabbitMQSource()
};
var mockResourceCatalog = new Mock <IResourceCatalog>();
mockResourceCatalog.Setup(o => o.FindByType(typeof(IQueueSource).FullName)).Returns(expected);
//------------Setup for test-------------------------
var service = new FindResourcesByType(new Lazy <IResourceCatalog>(() => mockResourceCatalog.Object));
IWorkspace workspace = null;
var values = new Dictionary <string, StringBuilder>();
values.Add("Type", new StringBuilder(typeof(IQueueSource).FullName));
//------------Execute Test---------------------------
var result = service.Execute(values, workspace);
//------------Assert Results-------------------------
var serializer = new Dev2JsonSerializer();
var expectedString = serializer.Serialize(expected, Newtonsoft.Json.Formatting.None);
Assert.AreEqual(expectedString, result.ToString());
}
public static IQueueSource <T> OrderedParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null)
{
Func <Tuple <int, T>, Task <Tuple <int, T, bool> > > func2 = async pair => new Tuple <int, T, bool>(pair.Item1, pair.Item2, await predicate(pair.Item2));
return(queue.Indexed().ParallelSelect(parallelWorker, func2, capacity).Reorder(pair => pair.Item1, 0, capacity).Where(i => Task.FromResult(i.Item3), capacity)
.SynchronousSelect(i => i.Item2));
}
public static IQueueSource <T> Where <T>(this IQueueSource <T> queue, Func <T, Task <bool> > predicate, int?capacity = null)
{
AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2);
Func <Task> worker = async delegate {
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (!item.HasValue)
{
break;
}
if (await predicate(item.Value))
{
await outQueue.Enqueue(item.Value, CancellationToken.None);
}
}
outQueue.WriteEof();
};
Task _dummy = Task.Run(worker);
return(outQueue);
}
public GetCancellableResult(IQueueSource <U> queue, U value, Func <U, V> convertResult)
{
this.queue = queue;
this.value = value;
this.convertResult = convertResult;
this.done = false;
}
public static CancellableOperationStarter <V> StartableGet <U, V>(this IQueueSource <U> queue, Func <U, V> convertResult, V eofResult)
{
return(delegate(CancellationToken ctoken)
{
CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ctoken);
Func <Task <CancellableResult <V> > > taskSource = async delegate()
{
AcquireReadResult arr = await queue.AcquireReadAsync(1, cts.Token);
if (arr is AcquireReadSucceeded <U> )
{
AcquireReadSucceeded <U> succeededResult = (AcquireReadSucceeded <U>)arr;
if (succeededResult.ItemCount == 1)
{
return new GetCancellableResult <U, V>(queue, succeededResult.Items[0], convertResult);
}
else
{
System.Diagnostics.Debug.Assert(succeededResult.ItemCount == 0);
return new GetEofCancellableResult <U, V>(queue, eofResult);
}
}
else if (arr is AcquireReadFaulted)
{
AcquireReadFaulted faultedResult = (AcquireReadFaulted)arr;
throw faultedResult.Exception;
}
else
{
AcquireReadCancelled cancelledResult = (AcquireReadCancelled)arr;
throw new OperationCanceledException();
}
};
Task <CancellableResult <V> > task = null;
try
{
task = Task.Run(taskSource);
}
catch (OperationCanceledException e)
{
task = Task.FromCanceled <CancellableResult <V> >(e.CancellationToken);
}
catch (Exception exc)
{
task = Task.FromException <CancellableResult <V> >(exc);
}
return new CancellableOperation <V>(task, cts);
});
}
public static IQueueSource <T> ParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null)
{
AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2);
IdleDetector idleDetector = new IdleDetector();
Func <Task> workPoster = async delegate()
{
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (!item.HasValue)
{
break;
}
T itemValue = item.Value;
idleDetector.Enter();
Task doingWork = await parallelWorker.StartWorkItem
(
async (int workerId) =>
{
try
{
if (await predicate(itemValue))
{
await outQueue.Enqueue(itemValue, CancellationToken.None);
}
}
finally
{
idleDetector.Leave();
}
},
CancellationToken.None
);
}
await idleDetector.WaitForIdle(CancellationToken.None);
outQueue.WriteEof();
};
Task.Run(workPoster);
return(outQueue);
}
public static async Task <Option <T> > Dequeue <T>(this IQueueSource <T> queue, CancellationToken ctoken)
{
AcquireReadResult result = await queue.AcquireReadAsync(1, ctoken);
return(result.Visit <Option <T> >
(
new Func <AcquireReadSucceeded, Option <T> >
(
succeeded =>
{
if (succeeded.ItemCount == 0)
{
return new None <T>();
}
else
{
System.Diagnostics.Debug.Assert(succeeded.ItemCount == 1);
bool wasCancelled = false;
if (ctoken.IsCancellationRequested)
{
wasCancelled = true;
queue.ReleaseRead(0);
}
else
{
queue.ReleaseRead(1);
}
if (wasCancelled)
{
throw new OperationCanceledException(ctoken);
}
return new Some <T>(((AcquireReadSucceeded <T>)succeeded).Items[0]);
}
}
),
new Func <AcquireReadCancelled, Option <T> >
(
cancelled => { throw new OperationCanceledException(ctoken); }
),
new Func <AcquireReadFaulted, Option <T> >
(
faulted => { throw faulted.Exception; }
)
));
}
public static IEnumerable <T> AsEnumerable <T>(this IQueueSource <T> queue)
{
BlockingCollection <T> bc = new BlockingCollection <T>();
Func <Task> feed = async delegate()
{
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (item.HasValue)
{
bc.Add(item.Value);
}
else
{
break;
}
}
bc.CompleteAdding();
};
Task _dummy = Task.Run(feed);
while (true)
{
T item2 = default(T);
bool success = false;
try
{
item2 = bc.Take();
success = true;
}
catch (InvalidOperationException exc)
{
// BlockingCollection ran out of items
System.Diagnostics.Debug.WriteLine(exc);
}
if (!success)
{
break;
}
yield return(item2);
}
}
public static Func <Task> ForEach <T>(IQueueSource <T> source, Func <ForEachInfo <T>, Task> processAsync, Func <Task> onCloseAsync, ExceptionCollector ec)
{
Func <Task> t = async delegate()
{
try
{
while (true)
{
var item = await source.Dequeue(ec.CancellationToken);
if (!item.HasValue)
{
break;
}
try
{
await processAsync(new ForEachInfo <T>(item.Value, 0, 0, ec.CancellationToken));
}
catch (Exception exc)
{
ec.Add(exc);
break;
}
}
}
finally
{
if (onCloseAsync != null)
{
try
{
await onCloseAsync();
}
catch (Exception exc)
{
ec.Add(exc);
}
}
}
};
return(t);
}
public static IQueueSource <T> Reorder <T>(this IQueueSource <T> queue, Func <T, int> getOrder, int first, int?capacity = null)
{
AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2);
Func <Task> worker = async delegate {
int next = first;
ImmutableDictionary <int, T> buffer = ImmutableDictionary <int, T> .Empty;
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (!item.HasValue)
{
break;
}
int order = getOrder(item.Value);
if (order == next)
{
await outQueue.Enqueue(item.Value, CancellationToken.None);
++next;
while (buffer.ContainsKey(next))
{
await outQueue.Enqueue(buffer[next], CancellationToken.None);
buffer = buffer.Remove(next);
++next;
}
}
else
{
buffer = buffer.Add(order, item.Value);
}
}
outQueue.WriteEof();
};
Task.Run(worker);
return(outQueue);
}
public IndexedSource(IQueueSource <T> parent)
{
this.parent = parent;
}
public GetEofCancellableResult(IQueueSource <U> queue, V eofResult)
{
this.queue = queue;
this.eofResult = eofResult;
this.done = false;
}
public static IQueueSource <U> OrderedParallelSelect <T, U>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <U> > func, int?capacity = null)
{
Func <Tuple <int, T>, Task <Tuple <int, U> > > func2 = async pair => new Tuple <int, U>(pair.Item1, await func(pair.Item2));
return(queue.Indexed().ParallelSelect(parallelWorker, func2, capacity).Reorder(pair => pair.Item1, 0, capacity).SynchronousSelect(pair => pair.Item2));
}
public static IQueueSource <U> SynchronousSelect <T, U>(this IQueueSource <T> queue, Func <T, U> func)
{
return(new SynchronousSelectedSource <T, U>(queue, func));
}
public SynchronousSelectedSource(IQueueSource <T> parent, Func <T, U> func)
{
this.parent = parent;
this.func = func;
}
public static IQueueSource <Tuple <int, T> > Indexed <T>(this IQueueSource <T> queue)
{
return(new IndexedSource <T>(queue));
}
