public CrawlerPropagator(
ITargetBlock <Submission> submissionOutput,
ITargetBlock <CrawlerError> errorOutput)
{
_submissionOutput = submissionOutput;
_errorOutput = errorOutput;
_input = InitInputBlock();
_queue = new Queue <CrawlerMessage>(BufferCapacity);
Completion = _input.Completion.ContinueWith(task =>
{
if (task.IsFaulted)
{
_submissionOutput.Fault(task.Exception);
_errorOutput.Fault(task.Exception);
}
else
{
// TODO: handle cancel
_submissionOutput.Complete();
_errorOutput.Complete();
}
});
}
/// <inheritdoc cref="ISourceBlock{TOutput}.LinkTo(ITargetBlock{TOutput}, DataflowLinkOptions)"/>
public IDisposable LinkTo(ITargetBlock <TOutput> target, DataflowLinkOptions linkOptions)
{
if (target == null)
{
throw new ArgumentNullException(nameof(target));
}
linkOptions ??= new DataflowLinkOptions();
LinkRegistration <TOutput> registration;
lock (this.Lock)
{
if (this._propagatedCompletion)
{
var exception = this.Completion.Exception;
if (exception == null)
{
target.Complete();
}
else
{
target.Fault(exception);
}
return(ActionDisposable.Nop);
}
registration = new LinkRegistration <TOutput>(target, linkOptions.MaxMessages, linkOptions.PropagateCompletion, this.HandleUnlink);
this._linkManager.AddLink(registration, linkOptions.Append);
}
this.OfferToTargets();
return(new ActionDisposable(registration.Unlink));
}
public static async Task SendAndCompleteAsync <T>(this ITargetBlock <T> target, IEnumerable <T> items, CancellationToken cancelToken = default(CancellationToken))
{
await Task.Yield();
try
{
foreach (var item in items)
{
if (target.Completion.IsFaulted)
{
throw target.Completion.Exception;
}
if (!await target.SendAsync(item, cancelToken))
{
throw new InvalidOperationException();
}
cancelToken.ThrowIfCancellationRequested();
}
target.Complete();
}
catch (Exception ex)
{
target.Fault(ex);
throw;
}
}
/// <summary>
/// Adds a target block to send messages to.
/// </summary>
/// <returns>
/// An object that can be used to destroy the link to the added target.
/// </returns>
public IDisposable AddTarget(ITargetBlock <T> targetBlock, DataflowLinkOptions options)
{
CancellationTokenSource cancellationTokenSource = null;
if (options.PropagateCompletion)
{
cancellationTokenSource = new CancellationTokenSource();
block.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
targetBlock.Fault(t.Exception);
}
else
{
targetBlock.Complete();
}
}, cancellationTokenSource.Token);
}
var target = new Target(
this, targetBlock, options.MaxMessages, cancellationTokenSource);
TargetDictionary [targetBlock] = target;
if (options.Append)
{
appendQueue.Enqueue(target);
}
else
{
prependQueue.Enqueue(target);
}
return(target);
}
/// <summary>
/// Links an action to this block. Usefull if it's an action block.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="target"></param>
/// <returns></returns>
public IFlowBlock BroadcastTo(ITargetBlock <TOut> target, bool linkToCompletion = true)
{
ItemProcessed += (x) =>
{
DataflowBlock.SendAsync(target, x).Wait();
};
if (linkToCompletion)
{
AddCompletionTask(target.Completion);
}
var actionBlock = this.GetProcessingBlock();
actionBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
target.Fault(t.Exception);
}
else
{
target.Complete();
}
});
//_lastLinkedBlock = target;
return(this);
}
IDisposable ISourceBlock <T> .LinkTo(ITargetBlock <T> target, DataflowLinkOptions?linkOptions)
{
if (target == null)
{
throw new ArgumentNullException(nameof(target));
}
linkOptions ??= new DataflowLinkOptions();
LinkRegistration <T> registration;
lock (this.CompletionLock)
{
if (this.Completion.IsCompleted)
{
var exception = this.Completion.Exception;
if (exception == null)
{
target.Complete();
}
else
{
target.Fault(exception);
}
return(ActionDisposable.Nop);
}
registration = new LinkRegistration <T>(target, linkOptions.MaxMessages, linkOptions.PropagateCompletion, this.HandleUnlink);
this._linkManager.AddLink(registration, linkOptions.Append);
}
this._resetEvent.Set();
return(new ActionDisposable(registration.Unlink));
}
/// <summary>
/// Watches the task, and shuts down the dataflow block (via <see cref="IDataflowBlock.Fault"/> or <see cref="IDataflowBlock.Complete"/>) when the task completes.
/// </summary>
/// <param name="task">The task to watch. May not be <c>null</c>.</param>
public void ObserveTaskForCompletion(Task task)
{
task.ContinueWith(_ =>
{
if (task.IsFaulted)
{
_block.Fault(task.Exception.InnerException);
}
else
{
_block.Complete();
}
}, TaskScheduler.Default);
}
/// <summary>
/// Runs when any of <see cref="_innerBlock"/> or <see cref="_outputBlock"/> faults
/// </summary>
/// <param name="faultedTask">Completion property of the faulted DataflowBlock</param>
private void OnAnyBlockFaulted(Task faultedTask)
{
// When one of the two DataflowBlocks faults, the faulted state is propagated to the following
// DataflowBlocks, but not to the preceding DataflowBlocks. In this case we therefore fault all DataflowBlocks
// that are not yet in a faulted state to ensure that all DataflowBlocks are completed and release their resources.
if (!_innerBlock.Completion.IsFaulted)
{
_innerBlock.Fault(faultedTask.Exception);
}
if (!_outputBlock.Completion.IsFaulted)
{
_outputBlock.Fault(faultedTask.Exception);
}
}
private static void LinkDataflowBlocks <TBlock>(ISourceBlock <TBlock> source, ITargetBlock <TBlock> target)
{
source.LinkTo(target);
source.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
target.Fault(t.Exception);
}
else
{
target.Complete();
}
});
}
public static void DeriveCompletionOrFaultFrom <TData>(this ITargetBlock <TData> target, IEnumerable <ISourceBlock <TData> > sources)
{
Task.WhenAll(sources.Select(x => x.Completion))
.ContinueWith(x =>
{
if (x.IsFaulted)
{
target.Fault(x.Exception);
}
else
{
target.Complete();
}
});
}
public async void Fault(Exception exception)
{
if (_target != null)
{
_target.Fault(exception);
}
//if (_source != null)
// _source.Fault(exception);
if (_target != null)
{
await _target.Completion;
}
//if (_source != null)
// await _source.Completion;
Dispose();
}
public static async Task WriteToBlockAsync <T>(this IAsyncEnumerable <T> enumerable,
ITargetBlock <T> block, CancellationToken token = default)
{
try
{
await foreach (var item in enumerable.WithCancellation(token).ConfigureAwait(false))
{
await block.SendAsync(item, token).ConfigureAwait(false);
}
block.Complete();
}
catch (Exception ex)
{
block.Fault(ex);
}
}
private async Task ReadAheadAsync()
{
Assumes.NotNull(this.readAheadElements);
try
{
while (await this.enumerator.MoveNextAsync().ConfigureAwait(false))
{
await this.readAheadElements.SendAsync(this.enumerator.Current, this.cancellationTokenSource.Token).ConfigureAwait(false);
}
this.readAheadElements.Complete();
}
catch (Exception ex)
{
ITargetBlock <T> target = this.readAheadElements;
target.Fault(ex);
}
}
private async Task ReadAheadAsync()
{
Assumes.NotNull(this.readAheadElements);
try
{
while (await this.enumerator.MoveNextAsync().ConfigureAwait(false))
{
await this.readAheadElements.SendAsync(this.enumerator.Current, this.cancellationTokenSource.Token).ConfigureAwait(false);
}
this.readAheadElements.Complete();
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception ex)
#pragma warning restore CA1031 // Do not catch general exception types
{
ITargetBlock <T> target = this.readAheadElements;
target.Fault(ex);
}
}
public async Task PostList(ITargetBlock <Core.Entities.BankAccount> target)
{
Diag.ThreadPrint("PostList - start");
var transform = new TransformBlock <BankAccount, Core.Entities.BankAccount>(ef =>
mapper.Map <Core.Entities.BankAccount>(ef), new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 4
});
transform.LinkTo(target);
await Task.Run(() =>
{
Diag.ThreadPrint("PostList - task start");
using (FinanceEntities context = factory.CreateContext())
{
(from b in context.BankAccounts.Include(a => a.Bank)
select b).AsParallel().ForAll(ef => transform.Post(ef));
//await transform.Completion;
transform.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
target.Fault(t.Exception);
}
else
{
Diag.ThreadPrint("PostList - task set target complete");
target.Complete();
}
});
transform.Complete();
}
Diag.ThreadPrint("PostList - task end");
}).ConfigureAwait(false);
Diag.ThreadPrint("PostList - end");
}
private async Task ReadMessageAsync(ITargetBlock <string> target)
{
string message = null;
var buffer = new byte[1024 * 4];
var segment = new ArraySegment <byte>(buffer);
using (var memoryStream = new MemoryStream())
{
try
{
WebSocketReceiveResult receiveResult;
do
{
receiveResult = await _socket.ReceiveAsync(segment, CancellationToken.None);
if (receiveResult.CloseStatus.HasValue)
{
target.Complete();
}
if (receiveResult.Count == 0)
{
continue;
}
await memoryStream.WriteAsync(segment.Array, segment.Offset, receiveResult.Count);
} while (!receiveResult.EndOfMessage || memoryStream.Length == 0);
message = Encoding.UTF8.GetString(memoryStream.ToArray());
}
catch (Exception x)
{
target.Fault(x);
}
}
await target.SendAsync(message);
}
public static void PostAndComplete <T>(this ITargetBlock <T> target, IEnumerable <T> items, CancellationToken cancelToken = default(CancellationToken))
{
try
{
foreach (var item in items)
{
if (!target.Post(item))
{
throw new InvalidOperationException();
}
cancelToken.ThrowIfCancellationRequested();
}
target.Complete();
}
catch (Exception ex)
{
target.Fault(ex);
throw;
}
}
/// <summary>
/// Links this block to a target block.
///
/// The target block receives all items accepted by the provided listener.
///
/// Completion of this block can optionally be propagated to the target block.
/// </summary>
public void LinkTo(ITargetBlock <TOutput> target, bool propagateCompletion = true)
{
Contract.Requires(target != null);
m_targetBlock = target;
if (propagateCompletion)
{
Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
Verbose("Propagating fault completion");
m_targetBlock.Fault(t.Exception.InnerException);
}
else
{
Verbose("Propagating completion");
m_targetBlock.Complete();
}
});
}
}
void IDataflowBlock.Fault(Exception exception) => consumer.Fault(exception);
public void OnError(Exception ex)
{
target.Fault(ex);
}
public void Fault(Exception ex)
{
compHelper.Fault(ex);
source.Fault(ex);
target.Fault(ex);
}
/// <summary>
/// Causes the <see cref="IDataflowBlock"/> to complete in a Faulted state.
/// </summary>
/// <param name="exception">The <see cref="Exception"/> that caused the faulting.</param>
/// <seealso cref="IDataflowBlock"/>
public void Fault(Exception exception)
{
_target.Fault(exception);
}
public void Fault(Exception exception)
{
targetBlock.Fault(exception);
}
public void Fault(Exception exception)
{
_targetBroadcastBlock.Fault(exception);
}
public void Fault(Exception exception)
{
_originalTargetBlock.Fault(exception);
}
public void Fault(Exception exception)
{
_bufferBlock.Fault(exception);
}
private async Task ReadMessageAsync(ITargetBlock <string> target)
{
while (!_socket.CloseStatus.HasValue)
{
string message;
byte[] buffer = new byte[1024 * 4];
var segment = new ArraySegment <byte>(buffer);
using (var memoryStream = new MemoryStream())
{
try
{
WebSocketReceiveResult receiveResult;
do
{
receiveResult = await _socket.ReceiveAsync(segment, CancellationToken.None);
if (receiveResult.CloseStatus.HasValue)
{
target.Complete();
}
if (receiveResult.Count == 0)
{
continue;
}
await memoryStream.WriteAsync(segment.Array, segment.Offset, receiveResult.Count);
} while (!receiveResult.EndOfMessage || memoryStream.Length == 0);
message = Encoding.UTF8.GetString(memoryStream.ToArray());
}
catch (WebSocketException wx)
{
WebSocketCloseStatus closeStatus;
switch (wx.WebSocketErrorCode)
{
case WebSocketError.ConnectionClosedPrematurely:
case WebSocketError.HeaderError:
case WebSocketError.UnsupportedProtocol:
case WebSocketError.UnsupportedVersion:
case WebSocketError.NotAWebSocket:
closeStatus = WebSocketCloseStatus.ProtocolError;
break;
case WebSocketError.InvalidMessageType:
closeStatus = WebSocketCloseStatus.InvalidMessageType;
break;
default:
closeStatus = WebSocketCloseStatus.InternalServerError;
break;
}
await Complete(closeStatus, $"Closing socket connection due to {wx.WebSocketErrorCode}.");
break;
}
catch (Exception x)
{
target.Fault(x);
continue;
}
}
await target.SendAsync(message);
}
}
public void Fault(Exception error)
{
m_target.Fault(error);
}
public void Fault(Exception exception)
{
_input.Fault(exception);
}
public void OnError(Exception error)
{
_block.Fault(error);
}
public async Task PostList(ITargetBlock<Core.Entities.BankAccount> target)
{
Diag.ThreadPrint("PostList - start");
var transform = new TransformBlock<BankAccount, Core.Entities.BankAccount>(ef =>
mapper.Map<Core.Entities.BankAccount>(ef), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 });
transform.LinkTo(target);
await Task.Run(() =>
{
Diag.ThreadPrint("PostList - task start");
using (FinanceEntities context = factory.CreateContext())
{
(from b in context.BankAccounts.Include(a => a.Bank)
select b).AsParallel().ForAll(ef => transform.Post(ef));
//await transform.Completion;
transform.Completion.ContinueWith(t =>
{
if (t.IsFaulted) target.Fault(t.Exception);
else
{
Diag.ThreadPrint("PostList - task set target complete");
target.Complete();
}
});
transform.Complete();
}
Diag.ThreadPrint("PostList - task end");
}).ConfigureAwait(false);
Diag.ThreadPrint("PostList - end");
}
