/// <summary>
/// Add a decorator to the enumerable for additional processing
/// </summary>
/// <param name="operation">The operation.</param>
/// <param name="enumerator">The enumerator.</param>
protected override IEnumerable <Row> DecorateEnumerableForExecution(IOperation operation, IEnumerable <Row> enumerator)
{
ThreadSafeEnumerator <Row> threadedEnumerator = new ThreadSafeEnumerator <Row>();
ThreadPool.QueueUserWorkItem(delegate
{
try
{
foreach (Row t in new EventRaisingEnumerator(operation, enumerator))
{
threadedEnumerator.AddItem(t);
}
}
catch (Exception e)
{
Error(e, "Failed to execute operation {0}", operation);
threadedEnumerator.MarkAsFinished();
#if DEBUG
throw e;
#endif
}
finally
{
threadedEnumerator.MarkAsFinished();
}
});
return(threadedEnumerator);
}
/// <summary>
/// Add a decorator to the enumerable for additional processing
/// </summary>
/// <param name="operation">The operation.</param>
/// <param name="enumerator">The enumerator.</param>
/// <param name="cancellationToken">A <see cref="T:System.Threading.CancellationToken" /> that may be used to cancel the asynchronous iteration.</param>
protected override AsyncEnumerableTask <Row> DecorateEnumerableForExecution(
IOperation operation,
IAsyncEnumerable <Row> enumerator,
CancellationToken cancellationToken = default)
{
ThreadSafeEnumerator <Row> threadedEnumerator = new ThreadSafeEnumerator <Row>();
Task task = Task.Run(async() =>
{
try
{
IAsyncEnumerable <Row> eventRaisingEnumerator = new EventRaisingEnumerator(operation, enumerator);
await eventRaisingEnumerator
.ForEachAsync(async t => { await threadedEnumerator.AddItem(t); },
cancellationToken: cancellationToken);
}
catch (Exception e)
{
Error(e, "Failed to execute operation {0}", new Tuple <string, object>("Operation", operation));
}
finally
{
await threadedEnumerator.MarkAsFinished();
}
}, cancellationToken);
return(new AsyncEnumerableTask <Row>
{
Enumerable = threadedEnumerator,
Task = task,
});
}
public void CanEnumerateSafely()
{
var strings = LangTestHelpers.RandomStrings(1000, 50);
var results = new ConcurrentQueue<string>();
using (var stringsEnumerator = strings.GetEnumerator())
{
var tse = new ThreadSafeEnumerator<string>(stringsEnumerator);
var threads = new List<Thread>();
for (var i = 0; i < 10; i++)
{
var thread = new Thread(() =>
{
string it = null;
while (tse.TryGetNext(ref it))
{
results.Enqueue(it);
}
});
thread.Start();
threads.Add(thread);
}
foreach (var thread in threads)
{
thread.Join();
}
}
CollectionAssert.AreEquivalent(strings, results);
}
public void CanEnumerateSafely()
{
var strings = LangTestHelpers.RandomStrings(1000, 50);
var results = new ConcurrentQueue <string>();
using (var stringsEnumerator = strings.GetEnumerator())
{
var tse = new ThreadSafeEnumerator <string>(stringsEnumerator);
var threads = new List <Thread>();
for (int i = 0; i < 10; i++)
{
var thread = new Thread(() =>
{
string it = null;
while (tse.TryGetNext(ref it))
{
results.Enqueue(it);
}
});
thread.Start();
threads.Add(thread);
}
foreach (var thread in threads)
{
thread.Join();
}
}
CollectionAssert.AreEquivalent(strings, results);
}
/// <summary>
/// Add a decorator to the enumerable for additional processing
/// </summary>
/// <param name="operation">The operation.</param>
/// <param name="enumerator">The enumerator.</param>
protected override IEnumerable<Row> DecorateEnumerableForExecution(IOperation operation, IEnumerable<Row> enumerator)
{
ThreadSafeEnumerator<Row> threadedEnumerator = new ThreadSafeEnumerator<Row>();
ThreadPool.QueueUserWorkItem(delegate
{
try
{
foreach (Row t in new EventRaisingEnumerator(operation, enumerator))
{
threadedEnumerator.AddItem(t);
}
}
catch (Exception e)
{
Error(e, "Failed to execute operation {0}", operation);
threadedEnumerator.MarkAsFinished();
throw;
}
finally
{
threadedEnumerator.MarkAsFinished();
}
});
return threadedEnumerator;
}
/// <summary>
/// Add a decorator to the enumerable for additional processing
/// </summary>
/// <param name="operation">The operation.</param>
/// <param name="enumerator">The enumerator.</param>
protected override IEnumerable <Row> DecorateEnumerableForExecution(IOperation operation, IEnumerable <Row> enumerator)
{
var threadedEnumerator = new ThreadSafeEnumerator <Row>();
ThreadPool.QueueUserWorkItem(delegate {
try {
foreach (Row t in new EventRaisingEnumerator(operation, enumerator))
{
threadedEnumerator.AddItem(t);
}
} catch (Exception e) {
foreach (var inner in e.FlattenHierarchy())
{
Error(inner, "Failed to execute {0}. {1}", operation.Name, inner.Message);
Debug(inner.StackTrace);
}
threadedEnumerator.MarkAsFinished();
#if DEBUG
throw new TransformalizeException(Logger, e.Message);
#endif
} finally {
threadedEnumerator.MarkAsFinished();
}
});
return(threadedEnumerator);
}
public virtual IEnumerable <TOutput> Execute(IEnumerable <TInput> input)
{
var threadSafe = new ThreadSafeEnumerator <TInput>();
var wrapper = new ChainedEnumeratorWrapper <TInput>(input, threadSafe);
return(operation.Execute(wrapper));
}
/// <summary>
/// Add a decorator to the enumerable for additional processing
/// </summary>
/// <param name="operation">The operation.</param>
/// <param name="enumerator">The enumerator.</param>
protected override IEnumerable<Row> DecorateEnumerableForExecution(IOperation operation, IEnumerable<Row> enumerator) {
var threadedEnumerator = new ThreadSafeEnumerator<Row>();
ThreadPool.QueueUserWorkItem(delegate {
try {
foreach (Row t in new EventRaisingEnumerator(operation, enumerator)) {
threadedEnumerator.AddItem(t);
}
} catch (Exception e) {
foreach (var inner in e.FlattenHierarchy()) {
Error(inner, "Failed to execute {0}. {1}", operation.Name, inner.Message);
Debug(inner.StackTrace);
}
threadedEnumerator.MarkAsFinished();
#if DEBUG
throw new TransformalizeException(Logger, e.Message);
#endif
} finally {
threadedEnumerator.MarkAsFinished();
}
});
return threadedEnumerator;
}
public void TestGetNextTransferItemsRetryGetChunks2()
{
var initialJobResponse = JobResponseStubs.BuildJobResponse(
JobResponseStubs.Chunk1(JobResponseStubs.NodeId1, false, false),
JobResponseStubs.Chunk2(JobResponseStubs.NodeId1, false, false),
JobResponseStubs.Chunk3(JobResponseStubs.NodeId1, false, false)
);
var node1Client = new Mock<IDs3Client>(MockBehavior.Strict);
var factory = new Mock<IDs3ClientFactory>(MockBehavior.Strict);
factory.Setup(cf => cf.GetClientForNodeId(JobResponseStubs.NodeId1)).Returns(node1Client.Object);
var client = new Mock<IDs3Client>(MockBehavior.Strict);
client
.Setup(c => c.BuildFactory(It.IsAny<IEnumerable<JobNode>>()))
.Returns(factory.Object);
var first = true;
client
.Setup(
c =>
c.GetJobChunksReadyForClientProcessingSpectraS3(
AllocateMock.AvailableChunks(JobResponseStubs.JobId)))
.Returns(() =>
{
if (first)
{
first = false;
return GetJobChunksReadyForClientProcessingSpectraS3Response.Success(TimeSpan.FromMinutes(5),
JobResponseStubs.BuildJobResponse(JobResponseStubs.Chunk1(JobResponseStubs.NodeId1, true,
true)));
}
return GetJobChunksReadyForClientProcessingSpectraS3Response.Success(TimeSpan.FromMinutes(5),
JobResponseStubs.BuildJobResponse(JobResponseStubs.Chunk2(JobResponseStubs.NodeId1, true, true)));
});
var source = new ReadStreamChunkStrategy(_ => { }, 5);
using (var transfers = source.GetNextTransferItems(client.Object, initialJobResponse).GetEnumerator())
{
var itemGetter = new ThreadSafeEnumerator<TransferItem>(transfers);
Assert.Throws<Ds3NoMoreRetriesException>(() =>
{
TransferItem it = null;
itemGetter.TryGetNext(ref it); //the first blob in the first chunk will return
source.CompleteBlob(it.Blob);
itemGetter.TryGetNext(ref it); //the second blob in the first chunk will return
source.CompleteBlob(it.Blob);
itemGetter.TryGetNext(ref it); //the first blob in the second chunk will return
source.CompleteBlob(it.Blob);
itemGetter.TryGetNext(ref it); //the second blob in the second chunk will return
source.CompleteBlob(it.Blob);
itemGetter.TryGetNext(ref it);
//exception will be thrown since we will get the same chunk over and over
});
}
node1Client.VerifyAll();
factory.VerifyAll();
}