public void TestBroadcastBlockConstructor()
{
try
{
// without option
var block = new BroadcastBlock<int>(i => i);
block = new BroadcastBlock<int>(null);
// There is no property to verify. If construction itself fails, there will be an exception.
//with not cancelled token and default scheduler
block = new BroadcastBlock<int>(i => i, new DataflowBlockOptions { MaxMessagesPerTask = 1 });
block = new BroadcastBlock<int>(null, new DataflowBlockOptions { MaxMessagesPerTask = 1 });
// There is no property to verify. If construction itself fails, there will be an exception.
//with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new BroadcastBlock<int>(i => i, new DataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token });
block = new BroadcastBlock<int>(null, new DataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token });
// There is no property to verify. If construction itself fails, there will be an exception.
}
catch (Exception ex)
{
Assert.True(false, "Test Failed, exception thrown during construction: " + ex.Message);
}
}
public void BasicUsageTest ()
{
bool act1 = false, act2 = false;
var evt = new CountdownEvent (2);
var broadcast = new BroadcastBlock<int> (null);
var action1 = new ActionBlock<int> (i =>
{
act1 = i == 42;
evt.Signal ();
});
var action2 = new ActionBlock<int> (i =>
{
act2 = i == 42;
evt.Signal ();
});
broadcast.LinkTo (action1);
broadcast.LinkTo (action2);
Assert.IsTrue (broadcast.Post (42));
Assert.IsTrue (evt.Wait (100));
Assert.IsTrue (act1);
Assert.IsTrue (act2);
}
public async Task TestPost()
{
var bb = new BroadcastBlock<int>(i => i);
Assert.True(bb.Post(1));
bb.Complete();
Assert.False(bb.Post(2));
await bb.Completion;
}
public void RegisterHandlers()
{
// add all handlers to the broadcast block so they receive news of events
_broadcastBlock = new BroadcastBlock<IGESEvent>(x => x);
// pass the two methods of the handler to the block, one determines if we are interested in this event, the other processes it.
_eventHandlers.ForEach(x => _broadcastBlock.LinkTo(x.ReturnActionBlock(), x.HandlesEvent));
_broadcastBlock.LinkTo(DataflowBlock.NullTarget<IGESEvent>());
}
public async Task TestReceiveThenPost()
{
var bb = new BroadcastBlock<int>(null);
var ignored = Task.Run(() => bb.Post(42));
Assert.Equal(expected: 42, actual: await bb.ReceiveAsync()); // this should always pass, but due to race we may not test what we're hoping to
bb = new BroadcastBlock<int>(null);
Task<int> t = bb.ReceiveAsync();
Assert.False(t.IsCompleted);
bb.Post(16);
Assert.Equal(expected: 16, actual: await t);
}
/// <summary>
/// 初期化処理。
/// </summary>
/// <param name="settingObject"></param>
/// <param name="token"></param>
public void Init(dynamic settingObject, CancellationToken token)
{
logger.Trace("Init Start");
var opt = new DataflowBlockOptions
{
CancellationToken = token,
};
var buffer = new BufferBlock<PastaLog>(opt);
var bloadcast = new BroadcastBlock<PastaLog>(CloneLog, opt);
buffer.LinkTo(bloadcast);
Target = buffer;
Source = bloadcast;
logger.Trace("Init End");
}
/// <summary>
/// Initializes a new instance of the <see cref="LogManager"/> class.
/// </summary>
/// <param name="loggingConfiguration">The logging configuration.</param>
public LogManager(LoggingConfiguration loggingConfiguration)
{
_LoggingConfiguration = loggingConfiguration;
_Broadcast = new BroadcastBlock<LogEntry>(
logEntry =>
logEntry.Clone(),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = loggingConfiguration.MaxDegreeOfParallelism <= 0
? Environment.ProcessorCount
: loggingConfiguration.MaxDegreeOfParallelism
});
}
private static BroadcastBlock<int> ConstructBroadcastNewWithNMessages(int messagesCount)
{
var block = new BroadcastBlock<int>(i => i);
for (int i = 0; i < messagesCount; i++)
{
block.Post(i);
}
// Wait until the messages have been properly buffered up.
// Otherwise TryReceive fails.
// Since there is no property to check on the BroadcastBlock, just sleep for 2 ms
Task.Delay(2).Wait();
return block;
}
public Statsd(string serviceName = null)
{
_log.Info("statsd.net starting.");
_tokenSource = new CancellationTokenSource();
_shutdownComplete = new ManualResetEvent(false);
SuperCheapIOC.Add(_log);
var systemInfoService = new SystemInfoService();
SuperCheapIOC.Add(systemInfoService as ISystemInfoService);
serviceName = serviceName ?? systemInfoService.HostName;
var systemMetricsService = new SystemMetricsService("statsdnet", serviceName);
SuperCheapIOC.Add(systemMetricsService as ISystemMetricsService);
/**
* The flow is:
* Listeners ->
* Message Parser ->
* router ->
* Aggregator ->
* Broadcaster ->
* Backends
*/
// Initialise the core blocks
_router = new StatsdMessageRouterBlock();
_messageParser = MessageParserBlockFactory.CreateMessageParserBlock(_tokenSource.Token,
SuperCheapIOC.Resolve<ISystemMetricsService>(),
_log);
_messageParser.LinkTo(_router);
_messageParser.Completion.LogAndContinueWith(_log, "MessageParser", () =>
{
_log.Info("MessageParser: Completion signaled. Notifying the MessageBroadcaster.");
_messageBroadcaster.Complete();
});
_messageBroadcaster = new BroadcastBlock<Bucket>(Bucket.Clone);
_messageBroadcaster.Completion.LogAndContinueWith(_log, "MessageBroadcaster", () =>
{
_log.Info("MessageBroadcaster: Completion signaled. Notifying all backends.");
_backends.ForEach(q => q.Complete());
});
// Add the broadcaster to the IOC container
SuperCheapIOC.Add<BroadcastBlock<Bucket>>(_messageBroadcaster);
systemMetricsService.SetTarget(_messageBroadcaster);
_backends = new List<IBackend>();
_listeners = new List<IListener>();
}
async Task GenerateFileFingerprintsAsync(CollectionPath[] paths, Func<FileInfo, bool> filePredicate,
ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> joinedTargetBlock,
CancellationToken cancellationToken)
{
try
{
var annotatedPathBroadcastBlock = new BroadcastBlock<AnnotatedPath[]>(aps => aps,
new DataflowBlockOptions { CancellationToken = cancellationToken });
var joiner = new LinkFingerprintJoiner(joinedTargetBlock);
annotatedPathBroadcastBlock.LinkTo(joiner.AnnotatedPathsBlock, new DataflowLinkOptions { PropagateCompletion = true });
var fileFingerprintBroadcastBlock = new BroadcastBlock<IFileFingerprint>(ff => ff,
new DataflowBlockOptions { CancellationToken = cancellationToken });
fileFingerprintBroadcastBlock.LinkTo(joiner.FileFingerprintBlock, new DataflowLinkOptions { PropagateCompletion = true });
var fingerprintGeneratorTask = _fileFingerprintManager
.GenerateFileFingerprintsAsync(annotatedPathBroadcastBlock, fileFingerprintBroadcastBlock, cancellationToken);
try
{
await DirectoryScanner.GenerateAnnotatedPathsAsync(paths, filePredicate, annotatedPathBroadcastBlock, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{ }
catch (Exception ex)
{
Console.Write("Path scan failed: " + ex.Message);
}
await fingerprintGeneratorTask.ConfigureAwait(false);
}
catch (Exception ex)
{
Console.WriteLine("GenerateFileFingerprintsAsync() failed: " + ex.Message);
}
finally
{
Debug.WriteLine("BlobManager.GenerateFileFingerprintsAsync() is done");
}
}
public async Task TestBroadcasting()
{
var bb = new BroadcastBlock<int>(i => i + 1);
var targets = Enumerable.Range(0, 3).Select(_ => new TransformBlock<int, int>(i => i)).ToArray();
foreach (var target in targets)
{
bb.LinkTo(target);
}
const int Messages = 3;
bb.PostRange(0, Messages);
for (int i = 0; i < Messages; i++)
{
foreach (var target in targets)
{
Assert.Equal(expected: i + 1, actual: await target.ReceiveAsync());
}
}
}
public void LinkAfterPostTest ()
{
bool act = false;
var evt = new ManualResetEventSlim ();
var broadcast = new BroadcastBlock<int> (null);
var action = new ActionBlock<int> (i =>
{
act = i == 42;
evt.Set ();
});
Assert.IsTrue (broadcast.Post (42));
broadcast.LinkTo (action);
Assert.IsTrue (evt.Wait (100));
Assert.IsTrue (act);
}
public MsBuildFileSystemWatcher(string directory, string filter, int delayMilliseconds, IFileSystem fileSystem, IMsBuildFileSystemFilter fileSystemFilter, TaskScheduler taskScheduler = null, IActionLog log = null) {
Requires.NotNullOrWhiteSpace(directory, nameof(directory));
Requires.NotNullOrWhiteSpace(filter, nameof(filter));
Requires.Range(delayMilliseconds >= 0, nameof(delayMilliseconds));
Requires.NotNull(fileSystem, nameof(fileSystem));
Requires.NotNull(fileSystemFilter, nameof(fileSystemFilter));
_directory = directory;
_filter = filter;
_delayMilliseconds = delayMilliseconds;
_fileSystem = fileSystem;
_fileSystemFilter = fileSystemFilter;
_taskScheduler = taskScheduler ?? TaskScheduler.Default;
_log = log ?? ProjectSystemActionLog.Default;
_queue = new ConcurrentQueue<IFileSystemChange>();
_broadcastBlock = new BroadcastBlock<Changeset>(b => b, new DataflowBlockOptions { TaskScheduler = _taskScheduler });
SourceBlock = _broadcastBlock.SafePublicize();
_fileSystemFilter.Seal();
}
public static void Example3()
{
// Scenario: one buffer and 3 actions connected to it - each number precessed is send to each action
var conf = new ExecutionDataflowBlockOptions();
//conf.MaxDegreeOfParallelism = 4;
var buffer = new BroadcastBlock<int>(i => i);
var action1 = new ActionBlock<int>(a =>
{
Thread.Sleep(50);
Console.WriteLine("Action 1 value: {0}", a);
}, conf);
var action2 = new ActionBlock<int>(a =>
{
Thread.Sleep(50);
Console.WriteLine("Action 2 value: {0}", a);
}, conf);
var action3 = new ActionBlock<int>(a =>
{
Thread.Sleep(50);
Console.WriteLine("Action 3 value: {0}", a);
}, conf);
buffer.LinkTo(action1);
buffer.LinkTo(action2);
buffer.LinkTo(action3);
var t = new Task(() =>
{
for (int i = 0; i < 12; i++)
{
buffer.Post(i);
}
});
t.Start();
}
public void Run()
{
var propagateCompletion = new DataflowLinkOptions { PropagateCompletion = true };
var csep = new ConcurrentExclusiveSchedulerPair();
var concurrentOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 2,
TaskScheduler = csep.ConcurrentScheduler
};
var exclusiveOptions = new ExecutionDataflowBlockOptions
{
TaskScheduler = csep.ExclusiveScheduler
};
var concurrent = new ActionBlock<int>(async i =>
{
Console.WriteLine("Concurent print value: {0}", i);
await Task.Delay(500);
}, concurrentOptions);
var exclusive = new ActionBlock<int>(i =>
{
Console.WriteLine("Exclusive print value: {0}", i);
Thread.Sleep(750);
}, exclusiveOptions);
var broadcaster = new BroadcastBlock<int>(i => i);
broadcaster.LinkTo(concurrent, propagateCompletion);
broadcaster.LinkTo(exclusive, propagateCompletion, i => i % 2 == 0);
Enumerable
.Range(1, 10)
.ToList()
.ForEach(i => broadcaster.Post(i));
broadcaster.Complete();
Task.WaitAll(concurrent.Completion, exclusive.Completion);
}
/// <summary>
/// Need to amke sure we cleanup the dataflow links and file watcher
/// </summary>
/// <param name="disposing"></param>
protected override void Dispose(bool disposing)
{
if (disposing)
{
CleanupFileWatcher();
if (FileChangeScheduler != null)
{
FileChangeScheduler.Dispose();
FileChangeScheduler = null;
}
if (_broadcastBlock != null)
{
_broadcastBlock.Complete();
_broadcastBlock = null;
}
if (_projectRuleSubscriptionLink != null)
{
_projectRuleSubscriptionLink.Dispose();
_projectRuleSubscriptionLink = null;
}
}
}
public void FaultConsumeBroadcast()
{
var scheduler = new TestScheduler();
var source = new BroadcastBlock <int> (null,
new DataflowBlockOptions {
TaskScheduler = scheduler
});
var target = new TestTargetBlock <int> {
Postpone = true
};
Assert.IsNotNull(source.LinkTo(target));
Assert.IsTrue(source.Post(42));
scheduler.ExecuteAll();
Assert.IsTrue(target.HasPostponed);
var exception = new Exception();
((IDataflowBlock)source).Fault(exception);
scheduler.ExecuteAll();
try {
source.Completion.Wait(1000);
Assert.Fail("Task must be faulted");
} catch (AggregateException ex) {
Assert.AreEqual(exception, ex.InnerException, "#9");
}
Assert.IsTrue(source.Completion.IsFaulted);
int value;
Assert.IsTrue(target.RetryPostponed(out value));
Assert.AreEqual(42, value);
}
public void CloningTest ()
{
object act1 = null, act2 = null;
var evt = new CountdownEvent (2);
object source = new object ();
var broadcast = new BroadcastBlock<object> (o => new object ());
var action1 = new ActionBlock<object> (i => { act1 = i; evt.Signal (); });
var action2 = new ActionBlock<object> (i => { act2 = i; evt.Signal (); });
broadcast.LinkTo (action1);
broadcast.LinkTo (action2);
Assert.IsTrue (broadcast.Post (source));
evt.Wait ();
Assert.IsNotNull (act1);
Assert.IsNotNull (act2);
Assert.IsFalse (source.Equals (act1));
Assert.IsFalse (source.Equals (act2));
Assert.IsFalse (act2.Equals (act1));
}
/// <summary>
/// 新增观察者
/// </summary>
/// <typeparam name="T"></typeparam>
public void AddObserver(object observer, string notificationName, Action <object> action)
{
ConcurrentDictionary <object, ActionBlock <object> > dicActionBlock;
if (!subscribeList.TryGetValue(notificationName, out dicActionBlock))
{
dicActionBlock = new ConcurrentDictionary <object, ActionBlock <object> >();
subscribeList.TryAdd(notificationName, dicActionBlock);
}
ActionBlock <object> handler;
if (dicActionBlock.TryGetValue(observer, out handler))
{
return;
}
handler = new ActionBlock <object>(obj =>
{
if (action != null)
{
action(obj);
}
});
dicActionBlock.TryAdd(observer, handler);
BroadcastBlock <object> broadcast;
if (!broadcastList.TryGetValue(notificationName, out broadcast))
{
broadcast = new BroadcastBlock <object>(msg => msg);
broadcastList.TryAdd(notificationName, broadcast);
}
broadcast.LinkTo(handler);
}
/// <summary>
/// The DebugProfileProvider sinks 2 sets of information
/// 1, Changes to the launchsettings.json file on disk
/// 2. Changes to the ActiveDebugProfile property in the .user file
/// </summary>
protected override void Initialize()
{
// Create our broadcast block for subscribers to get new ILaunchProfiles Information
_broadcastBlock = new BroadcastBlock <ILaunchSettings>(s => s);
_changedSourceBlock = _broadcastBlock.SafePublicize();
// Subscribe to changes to the broadcast block using the idle scheduler. This should filter out a lot of the intermediates
// states that files can be in.
if (ProjectSubscriptionService != null)
{
var projectChangesBlock = new ActionBlock <IProjectVersionedValue <IProjectSubscriptionUpdate> >(ProjectRuleBlock_ChangedAsync);
_projectRuleSubscriptionLink = ProjectSubscriptionService.ProjectRuleSource.SourceBlock.LinkTo(
projectChangesBlock,
ruleNames: ProjectDebugger.SchemaName,
linkOptions: new DataflowLinkOptions {
PropagateCompletion = true
});
}
// Make sure we are watching the file at this point
WatchLaunchSettingsFile();
}
/// <summary>
/// Create service
/// </summary>
/// <param name="connectionString"></param>
public IoTHubService(string connectionString = null)
{
if (string.IsNullOrEmpty(connectionString))
{
connectionString = Environment.GetEnvironmentVariable("_HUB_CS");
}
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentException("You must provide the iothubowner connection " +
"string, which can be obtained from the Azure IoT Hub portal. You must " +
"then either initialize the DefaultProvider class, or set the _HUB_CS " +
"environment variable.");
}
_hubConnectionString = ConnectionString.Parse(connectionString);
_updateQueue = new BufferBlock <Tuple <INameRecord, NameServiceOperation> >(
new DataflowBlockOptions {
NameFormat = "Update (in IoTHubService) Id={1}",
EnsureOrdered = false,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
CancellationToken = _open.Token
});
_listeners = new BroadcastBlock <Tuple <INameRecord, NameServiceEvent> >(null,
new DataflowBlockOptions {
NameFormat = "Notify (in IoTHubService) Id={1}",
EnsureOrdered = false,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded,
CancellationToken = _open.Token
});
Write = DataflowBlockEx.Encapsulate(_updateQueue, _listeners);
_registryLoaded = new TaskCompletionSource <bool>();
_cache = new ConcurrentDictionary <Reference, IoTHubRecord>();
_cacheWorker = CacheWorker();
}
public Logger(int maxBufferCapacity = 100, params ILogSink[] sinks)
{
_cancellationSource = new CancellationTokenSource();
_buffer = new BufferBlock <ILogEntry>(
new DataflowBlockOptions
{
BoundedCapacity = maxBufferCapacity,
CancellationToken = _cancellationSource.Token
});
_broadcast = new BroadcastBlock <ILogEntry>(entry => entry, new DataflowBlockOptions {
CancellationToken = _cancellationSource.Token
});
LinkDataflowBlocks(_buffer, _broadcast);
_sinks = new List <ActionBlock <ILogEntry> >();
foreach (var sink in sinks)
{
AddLogSink(sink);
}
}
public void BoundedQueuedTest ()
{
var scheduler = new TestScheduler ();
var broadcast = new BroadcastBlock<int> (
null,
new DataflowBlockOptions { TaskScheduler = scheduler, BoundedCapacity = 1 });
Assert.IsTrue (broadcast.Post (1));
Assert.IsFalse (broadcast.Post (2));
}
public void ReserveChangedTest ()
{
var scheduler = new TestScheduler ();
var broadcast = new BroadcastBlock<int> (null,
new DataflowBlockOptions { TaskScheduler = scheduler });
var target = new TestTargetBlock<int> { Postpone = true };
broadcast.LinkTo (target);
Assert.IsFalse (target.HasPostponed);
Assert.IsTrue (broadcast.Post (1));
scheduler.ExecuteAll ();
Assert.IsTrue (target.HasPostponed);
Assert.IsTrue(broadcast.Post(2));
scheduler.ExecuteAll ();
Assert.IsTrue (target.ReservePostponed ());
int value;
Assert.IsTrue (target.RetryPostponed (out value));
Assert.AreEqual (2, value);
}
static async void StartCrawling()
{
if (!Directory.Exists("Images"))
{
Directory.CreateDirectory("Images");
}
try
{
#region Dataflow block Options
var downloaderOptions = new ExecutionDataflowBlockOptions
{
// enforce fairness, after handling n messages
// the block's task will be re-schedule.
// this will give the opportunity for other block
// to actively process there messages (to avoid over subscription
// the Tpl dataflow does not schedule all task at once if the machine
// does not have enough cores)
MaxMessagesPerTask = DOWNLOADER_MAX_MESSAGE_PER_TASK,
// by default Tpl dataflow assign a single task per block,
// but you can control it by using the MaxDegreeOfParallelism
MaxDegreeOfParallelism = 2, //DOWNLOADER_MAX_DEGREE_OF_PARALLELISM,
// the size of the block input buffer
BoundedCapacity = DOWNLOADER_BOUNDED_CAPACITY
};
var transformerOptions = new ExecutionDataflowBlockOptions
{
MaxMessagesPerTask = MAX_MESSAGE_PER_TASK,
};
var writerOptions = new ExecutionDataflowBlockOptions
{
MaxMessagesPerTask = WRITER_MAX_DEGREE_OF_PARALLELISM,
MaxDegreeOfParallelism = DOWNLOADER_MAX_DEGREE_OF_PARALLELISM
};
var linkOption = new DataflowLinkOptions {
PropagateCompletion = true
};
#endregion // Dataflow block Options
#region Downloader
var downloader = new TransformBlock <string, XElement>(
async(url) =>
{
try
{
#region Validation
//if (_urls.ContainsKey(url))
// return null;
if (!_urls.TryAdd(url, true))
{
return(null);
}
#endregion // Validation
// using IOCP the thread pool worker thread does return to the pool
WebClient wc = new WebClient();
Task <string> download = wc.DownloadStringTaskAsync(url);
Task cancel = Task.Delay(DOWNLOAD_TIMEOUT_SEC * 1000);
Task any = await Task.WhenAny(download, cancel).ConfigureAwait(false);
#region Timeout validation
if (any == cancel)
{
wc.CancelAsync();
WriteToConsole("Cancel: [{0}]", ConsoleColor.Gray, url);
return(null);
}
#endregion // Timeout validation
string html = download.Result;
WriteToConsole("Downloaded: {0}", ConsoleColor.White, url);
XElement e = HtmlToXElement(html);
return(e);
}
#region Exception Handling
catch (WebException ex)
{
WriteToConsole("Error: [{0}]\r\n\t{1}", ConsoleColor.Red, url, ex.Message);
}
catch (AggregateException ex)
{
foreach (var exc in ex.Flatten().InnerExceptions)
{
WriteToConsole("Error: [{0}]\r\n\t{1}", ConsoleColor.Red, url, exc.Message);
}
}
catch (Exception ex)
{
WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message);
}
#endregion // Exception Handling
return(null);
}, downloaderOptions);
#endregion // Downloader
#region Link Parser
var linkParser = new TransformManyBlock <XElement, string>(
(html) =>
{
#region Validation
if (html == null)
{
return(Enumerable.Empty <string>());
}
#endregion // Validation
var linkes = from item in html.Descendants()
where item.Name.LocalName == "a"
let href = item.Attribute("href")
where href != null
select href.Value;
try
{
var result = linkes.ToArray();
return(result);
}
#region Exception Handling
catch (Exception ex)
{
WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message);
return(Enumerable.Empty <string>());
}
#endregion // Exception Handling
}, transformerOptions);
#endregion // Link Parser
#region Image Parser
var imgParser = new TransformManyBlock <XElement, string>(
(html) =>
{
var images = from item in html.Descendants()
where item.Name.LocalName == "img"
let src = item.Attribute("src")
where src != null
select src.Value;
try
{
var result = images.ToArray();
return(result);
}
#region Exception Handling
catch (Exception ex)
{
WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message);
return(Enumerable.Empty <string>());
}
#endregion // Exception Handling
}, transformerOptions);
#endregion // Image Parser
#region Writer
var writer = new ActionBlock <string>(async url =>
{
try
{
if (string.IsNullOrEmpty(url))
{
return;
}
WebClient wc = new WebClient();
// using IOCP the thread pool worker thread does return to the pool
byte[] buffer = await wc.DownloadDataTaskAsync(url).ConfigureAwait(false);
string fileName = Path.GetFileName(url);
#region Validation
if (!_images.TryAdd(fileName, true))
{
return;
}
#endregion // Validation
string name = @"Images\" + fileName;
using (var image = Image.FromStream(new MemoryStream(buffer)))
{
if (image.Width > MIN_SIZE.Width && image.Height > MIN_SIZE.Height)
{
using (Stream srm = OpenWriteAsync(name))
{
await srm.WriteAsync(buffer, 0, buffer.Length).ConfigureAwait(false);
WriteToConsole("{0}: Width:{1}, Height:{2}", ConsoleColor.Yellow,
fileName, image.Width, image.Height);
}
}
}
}
#region Exception Handling
catch (WebException ex)
{
WriteToConsole("Error: [{0}]\r\n\t{1}", ConsoleColor.Red, url, ex.Message);
}
catch (Exception ex)
{
WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message);
}
#endregion // Exception Handling
}, writerOptions);
#endregion // Writer
var garbageA = DataflowBlock.NullTarget <XElement>();
var garbageB = DataflowBlock.NullTarget <string>();
// use to broadcast the downloader output to the image and link parsers
var contentBroadcaster = new BroadcastBlock <XElement>(s => s);
var linkBroadcaster = new BroadcastBlock <string>(s => s);
#region LinkTo
///////////////////////////////////////////////////////////////////////
// garbage <------- downloader <------------------------- //
// | | //
// contentBroadcaster | //
// / \ | //
// imgParser linkParsers--->linkBroadcaster //
// | | //
// writer <----------------------------------------- //
///////////////////////////////////////////////////////////////////////
downloader.LinkTo(contentBroadcaster, linkOption, html => html != null);
downloader.LinkTo(garbageA); //, linkOption /*, html => html == null*/); // fallback (otherwise empty messages will be stack in the block buffer and the block will never complete)
contentBroadcaster.LinkTo(imgParser, linkOption);
contentBroadcaster.LinkTo(linkParser, linkOption);
linkParser.LinkTo(linkBroadcaster, linkOption);
StringComparison comparison = StringComparison.InvariantCultureIgnoreCase;
Predicate <string> linkFilter = link => link.StartsWith("http://");
Predicate <string> imgFilter = url =>
url.StartsWith("http://") &&
(url.EndsWith(".jpg", comparison) ||
url.EndsWith(".png", comparison) ||
url.EndsWith(".gif", comparison));
// Predicate<string> imgToGarbageFilter = url => !imgFilter(url);
imgParser.LinkTo(writer, linkOption, imgFilter);
imgParser.LinkTo(garbageB);// , imgToGarbageFilter);
linkBroadcaster.LinkTo(writer, linkOption, imgFilter);
linkBroadcaster.LinkTo(downloader, linkOption, linkFilter);
//linkBroadcaster.LinkTo(garbage);
#endregion // LinkTo
downloader.Post(URL_CRAWL_TARGET);
Console.WriteLine("Crawling");
Thread.Sleep(COMPLETE_AFTER_SEC * 1000);
#region Complete
downloader.Complete();
#region WriteToConsole ("Try to Complete...")
ConsoleColor color = ConsoleColor.Yellow;
WriteToConsole(
@"Try to Complete (items in the buffer =
downloader: is completed = {0}, input={1} , output={2}
writer: is completed = {3}, input ={4}
linkParser: is completed = {5}, input={6} , output={7}
imgParser: is completed = {8}, input={9} , output={10}
linkBroadcaster: is completed = {11},
contentBroadcaster: is completed = {12}", color,
downloader.Completion.IsCompleted, downloader.InputCount, downloader.OutputCount,
writer.Completion.IsCompleted, writer.InputCount,
linkParser.Completion.IsCompleted, linkParser.InputCount, linkParser.OutputCount,
imgParser.Completion.IsCompleted, imgParser.InputCount, imgParser.OutputCount,
linkBroadcaster.Completion.IsCompleted,
contentBroadcaster.Completion.IsCompleted);
#endregion // WriteToConsole ("Try to Complete...")
Task completeAll = Task.WhenAll(
downloader.Completion,
linkParser.Completion,
imgParser.Completion,
contentBroadcaster.Completion,
writer.Completion);
await Task.Run(async() =>
{
while (!completeAll.IsCompleted)
{
await Task.Delay(2000).ConfigureAwait(false);
#region WriteToConsole (status)
color = color == ConsoleColor.Magenta ? ConsoleColor.White : ConsoleColor.Yellow;
WriteToConsole(
@"Complete Status (items in the buffer =
downloader: is completed = {0}, input={1} , output={2}
writer: is completed = {3}, input ={4}
linkParser: is completed = {5}, input={6} , output={7}
imgParser: is completed = {8}, input={9} , output={10}
linkBroadcaster: is completed = {11},
contentBroadcaster: is completed = {12}
", color,
downloader.Completion.IsCompleted, downloader.InputCount, downloader.OutputCount,
writer.Completion.IsCompleted, writer.InputCount,
linkParser.Completion.IsCompleted, linkParser.InputCount, linkParser.OutputCount,
imgParser.Completion.IsCompleted, imgParser.InputCount, imgParser.OutputCount,
linkBroadcaster.Completion.IsCompleted,
contentBroadcaster.Completion.IsCompleted);
}
#endregion // WriteToConsole (status)
}).ConfigureAwait(false);
WriteToConsole("Complete (items in the writer input buffer = {0})", ConsoleColor.Green, writer.InputCount);
#endregion // Complete
}
catch (Exception ex)
{
WriteToConsole("EXCEPTION: {0}", ConsoleColor.DarkRed, ex);
}
}
public TPLDataflowSubscribers()
{
_jobs = new BroadcastBlock <IJob>(job => job);
}
public async Task TestFaultyScheduler()
{
var bb = new BroadcastBlock<int>(null, new DataflowBlockOptions {
BoundedCapacity = 1,
TaskScheduler = new DelegateTaskScheduler
{
QueueTaskDelegate = delegate { throw new FormatException(); }
}
});
Task<bool> t1 = bb.SendAsync(1);
Task<bool> t2 = bb.SendAsync(2);
bb.LinkTo(DataflowBlock.NullTarget<int>());
await Assert.ThrowsAsync<TaskSchedulerException>(() => bb.Completion);
Assert.True(await t1);
Assert.False(await t2);
}
public async Task TestBounding()
{
var bb = new BroadcastBlock<int>(null, new DataflowBlockOptions { BoundedCapacity = 1 });
var ab = new ActionBlock<int>(i => { });
bb.LinkTo(ab, new DataflowLinkOptions { PropagateCompletion = true });
Task<bool>[] sends = Enumerable.Range(0, 40).Select(i => bb.SendAsync(i)).ToArray();
bb.Complete();
await Task.WhenAll(sends);
await ab.Completion;
}
static void Main()
{
var rand = new Random(DateTime.Now.Millisecond);
var broadcastBlock = new BroadcastBlock <int>(x => x);
var transformPositive = new TransformBlock <int, int>(x =>
{
Thread.Sleep(1000);
return(x);
});
var transformNegative = new TransformBlock <int, int>(x =>
{
Thread.Sleep(2000);
return(-x);
});
var join = new JoinBlock <int, int>();
var batchBlock = new BatchBlock <Tuple <int, int> >(5);
var sumBlock = new ActionBlock <Tuple <int, int>[]>(tuples =>
{
foreach (var tuple in tuples)
{
Console.WriteLine($"{tuple.Item1}+({tuple.Item2})={tuple.Item1 + tuple.Item2}");
}
});
broadcastBlock.LinkTo(transformPositive, new DataflowLinkOptions {
PropagateCompletion = true
});
broadcastBlock.LinkTo(transformNegative, new DataflowLinkOptions {
PropagateCompletion = true
});
transformPositive.LinkTo(join.Target1, new DataflowLinkOptions {
PropagateCompletion = true
});
transformNegative.LinkTo(join.Target2, new DataflowLinkOptions {
PropagateCompletion = true
});
join.LinkTo(batchBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
batchBlock.LinkTo(sumBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
for (int i = 0; i < 30; i++)
{
broadcastBlock.Post(rand.Next(100));
Thread.Sleep(1000);
}
broadcastBlock.Complete();
Console.ReadLine();
}
static void Main(string[] args)
{
StringComparison comparison = StringComparison.InvariantCultureIgnoreCase;
Predicate <string> linkFilter = link =>
link.IndexOf(".aspx", comparison) != -1 ||
link.IndexOf(".php", comparison) != -1 ||
link.IndexOf(".htm", comparison) != -1 ||
link.IndexOf(".html", comparison) != -1 ||
link.EndsWith(".com", comparison) ||
link.EndsWith(".net", comparison);
Predicate <string> imgFilter = url =>
url.EndsWith(".jpg", comparison) ||
url.EndsWith(".png", comparison) ||
url.EndsWith(".gif", comparison);
var downloader = new TransformBlock <string, string>(
async(url) =>
{
// using IOCP the thread pool worker thread does return to the pool
var client = new HttpClient();
string result = await client.GetStringAsync(url);
return(result);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 2
});
var contentBroadcaster = new BroadcastBlock <string>(s => s);
var linkParser = new TransformManyBlock <string, string>(
(html) =>
{
var output = new List <string>();
var links = _linkRegexHRef.Matches(html);
foreach (Match item in links)
{
var value = item.Groups["LINK"].Value;
output.Add(value);
}
return(output);
});
var imgParser = new TransformManyBlock <string, string>(
(html) =>
{
var output = new List <string>();
var images = _imgRegex.Matches(html);
foreach (Match item in images)
{
var value = item.Groups["IMG"].Value;
output.Add(value);
}
return(output);
});
var writer = new ActionBlock <string>(async url =>
{
var client = new HttpClient();
// using IOCP the thread pool worker thread does return to the pool
byte[] buffer = await client.GetByteArrayAsync(url);
string fileName = Path.GetFileName(url);
Directory.CreateDirectory(AppDomain.CurrentDomain.BaseDirectory + @"\Images\");
string name = AppDomain.CurrentDomain.BaseDirectory + @"\Images\" + fileName;
using (Stream srm = File.OpenWrite(name))
{
await srm.WriteAsync(buffer, 0, buffer.Length);
}
});
var linkBroadcaster = new BroadcastBlock <string>(s => s);
IDisposable disposeAll = new CompositeDisposable(
// from [downloader] to [contentBroadcaster]
downloader.LinkTo(contentBroadcaster),
// from [contentBroadcaster] to [imgParser]
contentBroadcaster.LinkTo(imgParser),
// from [contentBroadcaster] to [linkParserHRef]
contentBroadcaster.LinkTo(linkParser),
// from [linkParser] to [linkBroadcaster]
linkParser.LinkTo(linkBroadcaster),
// conditional link to from [linkBroadcaster] to [downloader]
linkBroadcaster.LinkTo(downloader, linkFilter),
// from [linkBroadcaster] to [writer]
linkBroadcaster.LinkTo(writer, imgFilter),
// from [imgParser] to [writer]
imgParser.LinkTo(writer));
downloader.Post("http://www.reactivex.io");
Console.WriteLine("Press any key to exit...");
Console.ReadLine();
downloader.Complete();
}
public void Test()
{
var action = new ActionBlock <int[]>(x => { }, new ExecutionDataflowBlockOptions {
});
var batch = new BatchBlock <int>(10, new GroupingDataflowBlockOptions {
});
using var likBA = batch.LinkTo(action, new DataflowLinkOptions { }, x => x[0] > 1); //actionBlock gets 10 elements
batch.TriggerBatch(); //for leftover elements
var batchJoined = new BatchedJoinBlock <int[], string>(10, new GroupingDataflowBlockOptions {
});
batch.LinkTo(batchJoined.Target1, new DataflowLinkOptions {
}, ints => ints[0] > 0);
var action2 = new ActionBlock <Tuple <IList <int[]>, IList <string> > >(x => { });
using var linkBJA = batchJoined.LinkTo(action2, new DataflowLinkOptions { }, x => x.Item1.Count == x.Item2.Count);//it's 10 total so 7-3 or 0-10 will propagate
var broadcast = new BroadcastBlock <int>(cloningFunction: x => x, new DataflowBlockOptions {
});
var action3 = new ActionBlock <int>(x => { });
var action4 = new ActionBlock <int>(x => { });
broadcast.LinkTo(action3, new DataflowLinkOptions {
}, x => x > 0);
broadcast.LinkTo(action4, new DataflowLinkOptions {
}, x => x > 0); //both will gete same elements
var buffer = new BufferBlock <int>(new DataflowBlockOptions {
});
var action5 = new ActionBlock <int>(x => { });
buffer.LinkTo(action5, new DataflowLinkOptions {
}, x => x == 0);
var buffer1 = new BufferBlock <int>();
var join = new JoinBlock <int, string>(new GroupingDataflowBlockOptions {
});
buffer1.LinkTo(join.Target1);
var action6 = new ActionBlock <Tuple <int, string> >(tlp => { });
join.LinkTo(action6);
var transform = new TransformBlock <int, string>(x => x.ToString(), new ExecutionDataflowBlockOptions {
});
var action7 = new ActionBlock <string>(str => { });
transform.LinkTo(action7);
var transformMany = new TransformManyBlock <int, string>(
x => Enumerable.Range(0, x).Select(i => i.ToString()));//one recived to many output
var action8 = new ActionBlock <string>(str => { });
var writeOnce = new WriteOnceBlock <int>(cloningFunction: x => x, new DataflowBlockOptions {
}); // it gets and stores 1 element but gives clones to linked blocks?
var action9 = new ActionBlock <int>(x => { });
writeOnce.LinkTo(action9);
var bufferC1 = new BufferBlock <int>();
var bufferC2 = new BufferBlock <string>();
var actionIdx = DataflowBlock.Choose <int, string>(
bufferC1, x => Console.WriteLine(x + 1),
bufferC2, str => Console.WriteLine(str + ""));
var actionE1 = new ActionBlock <int>(x => { });
var bufferE1 = new BufferBlock <string>();
var propagator = DataflowBlock.Encapsulate <int, string>(actionE1, bufferE1);
var scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, Environment.ProcessorCount, int.MaxValue);
var concurrent = scheduler.ConcurrentScheduler;
var exclusive = scheduler.ExclusiveScheduler;
}
private static async Task Main(string[] args)
{
var baseUri = new Uri(args[0]);
var outputSize = args.Length > 1
? int.Parse(args[1])
: DefaultOutputSize;
var downloadString = new TransformBlock <Uri, string>(
async uri => {
using var client = new HttpClient();
return(await client.GetStringAsync(uri));
});
var visitedUris = new HashSet <Uri>();
var findUris = new TransformManyBlock <string, Uri>(
html => AnchorRef.Matches(html)
.Cast <Match>()
.Select(m => new Uri(m.Value, UriKind.RelativeOrAbsolute))
.Select(uri => uri.IsAbsoluteUri ? uri : new Uri(baseUri, uri))
.Where(uri => visitedUris.Add(uri)),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
});
var broadcastUris = new BroadcastBlock <Uri>(uri => uri);
var collectResults = new BatchBlock <Uri>(outputSize);
downloadString.LinkTo(findUris);
findUris.LinkTo(broadcastUris);
broadcastUris.LinkTo(collectResults);
broadcastUris.LinkTo(
downloadString,
uri => baseUri.IsBaseOf(uri) && uri != baseUri);
downloadString.Post(baseUri);
try
{
var uris = await collectResults.ReceiveAsync(TimeSpan.FromSeconds(5));
PrintResults(uris);
}
catch (InvalidOperationException exception) // time out
{
PrintException(exception);
}
finally
{
downloadString.Complete();
}
try
{
await downloadString.Completion;
}
catch (AggregateException exception)
{
PrintException(exception);
}
catch (Exception exception)
{
PrintException(exception);
}
}
public flow()
{
m_buffer = new BufferBlock <object>();
BroadCast = new BroadcastBlock <string>(str => str);
}
public void TryReceiveTest()
{
var scheduler = new TestScheduler();
var block = new BroadcastBlock<int>(i => i * 10, new DataflowBlockOptions { TaskScheduler = scheduler });
int item;
Assert.IsFalse(block.TryReceive(null, out item));
Assert.IsTrue(block.Post(1));
Assert.IsTrue(block.Post(2));
scheduler.ExecuteAll();
Assert.IsTrue(block.TryReceive(null, out item));
Assert.AreEqual(20, item);
// predicate is tested on original value, but returned is cloned
Assert.IsTrue(block.TryReceive(i => i < 10, out item));
Assert.AreEqual(20, item);
}
public ProviderManagerFlow(
ILogger <ProviderManagerFlow> logger,
IMessageDumper messageDumper,
HttpClientManager httpClientManager,
IHttpRequestMessageFactory httpRequestMessageFactory,
ITranslationDistributer translationDistributer
)
{
this.logger = logger;
this.messageDumper = messageDumper;
this.httpClientManager = httpClientManager;
this.httpRequestMessageFactory = httpRequestMessageFactory;
this.translationDistributer = translationDistributer;
createHttpRequestBlock
= new TransformManyBlock <MessageContext <TranslationsRequest>, MessageContext <HttpRequestMessage> >(
CreateHttpRequestHandler,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
}
);
execHttpRequestBlock
= new TransformManyBlock <MessageContext <HttpRequestMessage>, MessageContext <HttpResponseMessage> >(
ExecHttpRequestHandler,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
}
);
parseHttpResponseBlock
= new TransformManyBlock <MessageContext <HttpResponseMessage>, MessageContext <string> >(
ParseHttpResponseHandler,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
}
);
broadcastResponseBlock = new BroadcastBlock <MessageContext <string> >(t => t);
dumpResponseBlock
= new ActionBlock <MessageContext <string> >(
DumpResponseHandler,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
}
);
deserializeResponseBlock
= new TransformManyBlock <MessageContext <string>, MessageContext <Translation> >(
DeserializeResponseHandler,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
}
);
translationDistributerBlock
= new ActionBlock <MessageContext <Translation> >(
TranslationDistributerHandler,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
}
);
createHttpRequestBlock.LinkTo(execHttpRequestBlock);
execHttpRequestBlock.LinkTo(parseHttpResponseBlock);
parseHttpResponseBlock.LinkTo(broadcastResponseBlock);
broadcastResponseBlock.LinkTo(dumpResponseBlock);
broadcastResponseBlock.LinkTo(deserializeResponseBlock);
deserializeResponseBlock.LinkTo(translationDistributerBlock);
}
public Task StartAsync(CancellationToken cancellationToken)
{
this.m_logger.LogDebug("Starting imagegrabber service");
this.m_cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
this.m_cts.Token.Register(() =>
{
this.m_logger.LogDebug("ImageGrabberService cancellation requested");
});
this.m_inputBuffer = new BufferBlock <ImageData>(new DataflowBlockOptions {
CancellationToken = this.m_cts.Token
});
this.m_transform = new TransformBlock <ImageData, ImageData>(new Func <ImageData, ImageData>(this.TransformImage), new ExecutionDataflowBlockOptions {
CancellationToken = this.m_cts.Token
});
this.m_output = new BroadcastBlock <ImageData>(img => img.Clone());
this.m_imageLogger = new ActionBlock <ImageData>(img =>
{
});
this.m_inputBuffer.LinkTo(this.m_transform);
this.m_inputBuffer.Completion.ContinueWith(t =>
{
this.m_logger.LogDebug("input buffer completed");
if (t.IsFaulted)
{
((IDataflowBlock)this.m_transform).Fault(t.Exception);
}
else
{
this.m_transform.Complete();
}
});
this.m_transform.LinkTo(this.m_output);
this.m_transform.Completion.ContinueWith(t =>
{
this.m_logger.LogDebug("transform block completed");
if (t.IsFaulted)
{
((IDataflowBlock)this.m_output).Fault(t.Exception);
}
else
{
this.m_output.Complete();
}
});
this.m_output.LinkTo(this.m_imageLogger);
this.m_output.Completion.ContinueWith(t =>
{
this.m_logger.LogDebug("output block completed");
if (t.IsFaulted)
{
((IDataflowBlock)this.m_imageLogger).Fault(t.Exception);
}
else
{
this.m_imageLogger.Complete();
}
});
try
{
this.m_arial = SystemFonts.Find("arial");
}
catch (FontFamilyNotFountException)
{
int count = SystemFonts.Collection.Families.Count();
this.m_logger.LogWarning($"Arial not found. {count} fonts available, taking first");
this.m_arial = SystemFonts.Collection.Families.FirstOrDefault();
if (count > 0)
{
foreach (var ff in SystemFonts.Collection.Families)
{
this.m_logger.LogInformation($"Font : {ff.Name}");
}
}
}
this.m_th.Start(this.m_cts.Token);
return(Task.CompletedTask);
}
public async Task TestFaultingAndCancellation()
{
foreach (bool fault in DataflowTestHelpers.BooleanValues)
{
var cts = new CancellationTokenSource();
var bb = new BroadcastBlock<int>(null, new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = 2 });
Task<bool>[] sends = Enumerable.Range(0, 4).Select(i => bb.SendAsync(i)).ToArray();
if (fault)
{
Assert.Throws<ArgumentNullException>(() => ((IDataflowBlock)bb).Fault(null));
((IDataflowBlock)bb).Fault(new InvalidCastException());
await Assert.ThrowsAsync<InvalidCastException>(() => bb.Completion);
}
else
{
cts.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion);
}
await Task.WhenAll(sends);
}
}
private static async Task Main(string[] args)
{
var bufferBlock = new BroadcastBlock <int>(a => a);
var a1 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Message {a} was processed by Consumer 1");
if (a % 2 == 0)
{
Task.Delay(300).Wait();
}
else
{
Task.Delay(50).Wait();
}
return(a * -1);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
}
);
var a2 = new TransformBlock <int, int>(a =>
{
Console.WriteLine($"Message {a} was processed by Consumer 2");
if (a % 2 == 0)
{
Task.Delay(50).Wait();
}
else
{
Task.Delay(300).Wait();
}
return(a);
}
, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
}
);
bufferBlock.LinkTo(a1);
bufferBlock.LinkTo(a2);
var joinBlock = new BatchedJoinBlock <int, int>(3);
a1.LinkTo(joinBlock.Target1);
a2.LinkTo(joinBlock.Target2);
var finalBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(a =>
{
Console.WriteLine($"Message {a.Item1},{a.Item2} was processed by all consumers. Total was [A=={string.Join(",", a.Item1)}:::::B=={string.Join(",", a.Item2)}] ");
});
joinBlock.LinkTo(finalBlock);
for (var i = 0; i < 10; i++)
{
await bufferBlock.SendAsync(i);
}
Console.WriteLine("Finished!");
Console.ReadKey();
}
public GuaranteedDeliveryBroadcastBlock(Func <T, T> cloningFunction)
{
_BroadcastBlock = new BroadcastBlock <T>(cloningFunction);
_Completion = _BroadcastBlock.Completion;
}
public async Task TestCloning()
{
// Test cloning when a clone function is provided
{
int data = 42;
var bb = new BroadcastBlock <int>(x => - x);
Assert.True(bb.Post(data));
for (int i = 0; i < 3; i++)
{
Assert.Equal(expected: -data, actual: await bb.ReceiveAsync());
Assert.Equal(expected: -data, actual: bb.Receive());
Assert.Equal(expected: -data, actual: await bb.ReceiveAsync());
IList <int> items;
Assert.True(((IReceivableSourceBlock <int>)bb).TryReceiveAll(out items));
Assert.Equal(expected: 1, actual: items.Count);
Assert.Equal(expected: -data, actual: items[0]);
}
int result = 0;
var target = new ActionBlock <int>(i => {
Assert.Equal(expected: 0, actual: result);
result = i;
Assert.Equal(expected: -data, actual: i);
});
bb.LinkTo(target, new DataflowLinkOptions {
PropagateCompletion = true
});
bb.Complete();
await target.Completion;
}
// Test successful processing when no clone function exists
{
var data = new object();
var bb = new BroadcastBlock <object>(null);
Assert.True(bb.Post(data));
object result;
for (int i = 0; i < 3; i++)
{
Assert.Equal(expected: data, actual: await bb.ReceiveAsync());
Assert.Equal(expected: data, actual: bb.Receive());
Assert.Equal(expected: data, actual: await bb.ReceiveAsync());
IList <object> items;
Assert.True(((IReceivableSourceBlock <object>)bb).TryReceiveAll(out items));
Assert.Equal(expected: 1, actual: items.Count);
Assert.Equal(expected: data, actual: items[0]);
}
result = null;
var target = new ActionBlock <object>(o => {
Assert.Null(result);
result = o;
Assert.Equal(expected: data, actual: o);
});
bb.LinkTo(target, new DataflowLinkOptions {
PropagateCompletion = true
});
bb.Complete();
await target.Completion;
}
}
public Multicast()
{
TypeInfo = new TypeInfo(typeof(TInput));
ObjectCopy = new ObjectCopy <TInput>(TypeInfo);
BroadcastBlock = new BroadcastBlock <TInput>(Clone);
}
public async Task SyncPathsAsync(string bucket, IEnumerable <string> paths, Func <FileInfo, bool> filePredicate, CancellationToken cancellationToken)
{
// BEWARE: This could cause trouble if there are
// any case-sensitive paths involved.
var namedPaths = (from arg in paths
let split = arg.IndexOf('=')
let validSplit = split > 0 && split < arg.Length - 1
select new CollectionPath(validSplit ? arg.Substring(0, split) : null,
PathUtil.ForceTrailingSlash(Path.GetFullPath(validSplit ? arg.Substring(split + 1) : arg))))
.Distinct()
.ToArray();
using (var blobManager = new BlobManager(new FileFingerprintManager(new MessagePackFileFingerprintStore(), new StreamFingerprinter())) as IBlobManager)
{
try
{
using (var awsManager = AwsManagerFactory.Create(bucket))
{
var uniqueFingerprintBlock = new BufferBlock <Tuple <FileFingerprint, AnnotatedPath> >();
var linkBlock = new BufferBlock <Tuple <AnnotatedPath, FileFingerprint> >();
var uniqueFingerprints = new HashSet <BlobFingerprint>();
var uniqueFingerprintFilterBlock = new ActionBlock <Tuple <AnnotatedPath, FileFingerprint> >(
t =>
{
if (!uniqueFingerprints.Add(t.Item2.Fingerprint))
{
return(Task.CompletedTask);
}
return(uniqueFingerprintBlock.SendAsync(Tuple.Create(t.Item2, t.Item1), cancellationToken));
}, new ExecutionDataflowBlockOptions {
CancellationToken = cancellationToken
});
var uniqueCompletionTask = uniqueFingerprintFilterBlock
.Completion.ContinueWith(_ => { uniqueFingerprintBlock.Complete(); }, CancellationToken.None);
TaskCollector.Default.Add(uniqueCompletionTask, "Unique filter completion");
var joinedBroadcastBlock = new BroadcastBlock <Tuple <AnnotatedPath, FileFingerprint> >(t => t,
new DataflowBlockOptions {
CancellationToken = cancellationToken
});
joinedBroadcastBlock.LinkTo(linkBlock, DataflowLinkOptionsPropagateEnabled);
joinedBroadcastBlock.LinkTo(uniqueFingerprintFilterBlock, DataflowLinkOptionsPropagateEnabled);
var tasks = new List <Task>();
var loadBlobTask = blobManager.LoadAsync(namedPaths, filePredicate, joinedBroadcastBlock, cancellationToken);
tasks.Add(loadBlobTask);
if (S3Settings.UpdateLinks)
{
var updateLinksTask = _s3LinkCreator.UpdateLinksAsync(awsManager, linkBlock, cancellationToken);
tasks.Add(updateLinksTask);
}
Task uploadBlobsTask = null;
var scanBlobAsync = Task.Run(async() =>
{
// ReSharper disable once AccessToDisposedClosure
var knownObjects = await awsManager.ScanAsync(cancellationToken).ConfigureAwait(false);
if (S3Settings.UploadBlobs)
{
// ReSharper disable once AccessToDisposedClosure
uploadBlobsTask = _s3BlobUploader.UploadBlobsAsync(awsManager, uniqueFingerprintBlock, knownObjects, cancellationToken);
}
}, cancellationToken);
tasks.Add(scanBlobAsync);
await WaitAllWithWake(tasks).ConfigureAwait(false);
if (null != uploadBlobsTask)
{
await uploadBlobsTask.ConfigureAwait(false);
}
}
}
finally
{
await blobManager.ShutdownAsync(cancellationToken).ConfigureAwait(false);
}
}
}
public void DontOfferTwiceTest()
{
var scheduler = new TestScheduler ();
var block = new BroadcastBlock<int> (null,
new DataflowBlockOptions { TaskScheduler = scheduler });
var target =
new TestTargetBlock<int> { Postpone = true };
block.LinkTo (target);
Assert.IsFalse (target.HasPostponed);
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll();
Assert.IsTrue (target.HasPostponed);
target.Postpone = false;
int value;
Assert.IsTrue(target.RetryPostponed(out value));
Assert.AreEqual(1, value);
block.LinkTo(new BufferBlock<int>());
scheduler.ExecuteAll();
Assert.AreEqual(default(int), target.DirectlyAccepted);
}
public void Exec()
{
Console.WriteLine("start");
// Downloads the requested resource as a string.
var createNumbers = new TransformManyBlock <int, int>(count => CreateNumbersFunc(count));
var printNumber = new ActionBlock <int>(number => PrintNumberFunc(number));
var printSecondFormat = new ActionBlock <int>(number =>
{
Thread.Sleep(2000);
if (number == 4 || number == 5)
{
throw new Exception("new error");
}
Console.WriteLine($" Second format {number}");
});
var errorReportBlock = new ActionBlock <int>(number => { Console.WriteLine($" Error on: {number}"); });
//1.work in parralel
//var printNumber = new ActionBlock<int>(number => PrintNumberFunc(number),new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 3 });
//2.filter mesages
//createNumbers.LinkTo(printNumber,n=>n%2==0 );
//createNumbers.LinkTo(printSecondFormat,n => n % 2 != 0);
//3.work in 2 blocks
var broadcast = new BroadcastBlock <int>(null);
createNumbers.LinkTo(broadcast);
broadcast.LinkTo(printSecondFormat);
broadcast.LinkTo(printNumber);
createNumbers.Completion.ContinueWith(t => broadcast.Complete());
broadcast.Completion.ContinueWith(t => printNumber.Complete());
createNumbers.Completion.ContinueWith(t => printSecondFormat.Complete());
createNumbers.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)printSecondFormat).Fault(t.Exception);
}
else
{
printSecondFormat.Complete();
}
});
Stopwatch sw = new Stopwatch();
sw.Start();
createNumbers.Post(1);
try
{
createNumbers.Complete();
printSecondFormat.Completion.Wait();
printNumber.Completion.Wait();
sw.Stop();
Console.WriteLine("time " + sw.Elapsed.TotalSeconds);
}
catch (AggregateException ex)
{
Console.WriteLine(ex.Flatten().Message);
}
Console.Read();
}
private void LoadAggregators(StatsdnetConfiguration config,
IntervalService intervalService,
BroadcastBlock <Bucket> messageBroadcaster,
ISystemMetricsService systemMetrics)
{
foreach (var aggregator in config.Aggregators)
{
switch (aggregator.Key)
{
case "counters":
var counter = aggregator.Value as CounterAggregationConfig;
AddAggregator(MessageType.Counter,
TimedCounterAggregatorBlockFactory.CreateBlock(messageBroadcaster,
counter.Namespace,
intervalService),
systemMetrics);
break;
case "gauges":
var gauge = aggregator.Value as GaugeAggregatorConfig;
AddAggregator(MessageType.Gauge,
TimedGaugeAggregatorBlockFactory.CreateBlock(messageBroadcaster,
gauge.Namespace,
gauge.RemoveZeroGauges,
intervalService),
systemMetrics);
break;
case "calendargrams":
var calendargram = aggregator.Value as CalendargramAggregationConfig;
AddAggregator(MessageType.Calendargram,
TimedCalendargramAggregatorBlockFactory.CreateBlock(messageBroadcaster,
calendargram.Namespace,
intervalService,
new TimeWindowService()),
systemMetrics);
break;
case "timers":
var timer = aggregator.Value as TimersAggregationConfig;
AddAggregator(MessageType.Timing,
TimedLatencyAggregatorBlockFactory.CreateBlock(messageBroadcaster,
timer.Namespace,
intervalService,
timer.CalculateSumSquares),
systemMetrics);
// Add Percentiles
foreach (var percentile in timer.Percentiles)
{
AddAggregator(MessageType.Timing,
TimedLatencyPercentileAggregatorBlockFactory.CreateBlock(messageBroadcaster,
timer.Namespace,
intervalService,
percentile.Threshold,
percentile.Name),
systemMetrics);
}
break;
}
}
// Add the Raw (pass-through) aggregator
AddAggregator(MessageType.Raw,
PassThroughBlockFactory.CreateBlock(messageBroadcaster, intervalService),
systemMetrics);
}
public TracingTPLPipeline(ILogger <TracingTPLPipeline> logger)
{
_BatchBlock = new BroadcastBlock <TracingBatch>(x => x);
_Logger = logger;
}
public async Task TestChoose_ConsumeToAccept()
{
var bb = new BroadcastBlock<int>(i => i * 2);
Task<int> t = DataflowBlock.Choose(bb, i => { }, bb, i => { });
Assert.False(t.IsCompleted);
bb.Post(42);
await t;
}
public static Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]> Generate3CrossGraph(int cntNodes,
Func <long[], long> f)
{
int cntRows = cntNodes;
int cntCols = cntNodes;
IPropagatorBlock <long, long>[,] matrix = new IPropagatorBlock <long, long> [cntRows, cntCols];
Stopwatch sw = new Stopwatch();
sw.Start();
// creation
for (int row = 0; row < cntRows; row++)
{
if (row == 0)
{
for (int col = 0; col < cntCols; col++)
{
matrix[row, col] = new BroadcastBlock <long>(x => x);
}
}
if (row > 0)
{
for (int col = 0; col < cntCols; col++)
{
if (col == 0 || col == cntCols - 1)
{
matrix[row, col] = Create3CrossNode(2, f);
}
else
{
matrix[row, col] = Create3CrossNode(3, f);
}
}
}
}
// linking
for (int row = 0; row < cntRows - 1; row++)
{
for (int col = 0; col < cntCols; col++)
{
if (col == 0)
{
matrix[row, col].LinkTo(matrix[row + 1, col]);
matrix[row, col].LinkTo(matrix[row + 1, col + 1]);
}
if (col > 0 && col < cntCols - 1)
{
matrix[row, col].LinkTo(matrix[row + 1, col - 1]);
matrix[row, col].LinkTo(matrix[row + 1, col]);
matrix[row, col].LinkTo(matrix[row + 1, col + 1]);
}
if (col == cntCols - 1)
{
matrix[row, col].LinkTo(matrix[row + 1, col - 1]);
matrix[row, col].LinkTo(matrix[row + 1, col]);
}
}
}
// first layer
var firstLayer = new IPropagatorBlock <long, long> [cntCols];
for (int i = 0; i < cntCols; i++)
{
firstLayer[i] = matrix[0, i];
}
//last layer
var lastLayer = new IPropagatorBlock <long, long> [cntCols];
for (int i = 0; i < cntCols; i++)
{
lastLayer[i] = matrix[cntRows - 1, i];
}
sw.Stop();
Console.WriteLine("Creation time: {0}", sw.Elapsed);
return(new Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]>(firstLayer, lastLayer));
}
public void QueuedMessagesTest ()
{
var scheduler = new TestScheduler ();
var broadcast = new BroadcastBlock<int> (null,
new DataflowBlockOptions { TaskScheduler = scheduler });
var target = new BufferBlock<int> ();
broadcast.LinkTo (target);
Assert.IsTrue (broadcast.Post (1));
Assert.IsTrue (broadcast.Post (2));
AssertEx.Throws<TimeoutException> (
() => target.Receive (TimeSpan.FromMilliseconds (100)));
scheduler.ExecuteAll ();
int item;
Assert.IsTrue (target.TryReceive (out item));
Assert.AreEqual (1, item);
Assert.IsTrue (target.TryReceive (out item));
Assert.AreEqual (2, item);
}
public async Task TestPrecancellation()
{
var b = new BroadcastBlock<int>(null, new DataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) });
Assert.NotNull(b.LinkTo(DataflowBlock.NullTarget<int>()));
Assert.False(b.Post(42));
Task<bool> t = b.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
int ignoredValue;
IList<int> ignoredValues;
Assert.False(b.TryReceive(out ignoredValue));
Assert.False(((IReceivableSourceBlock<int>)b).TryReceiveAll(out ignoredValues));
Assert.NotNull(b.Completion);
b.Complete(); // verify doesn't throw
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => b.Completion);
}
public void BoundedPostponedTest ()
{
var scheduler = new TestScheduler ();
var broadcast = new BroadcastBlock<int> (
null,
new DataflowBlockOptions { TaskScheduler = scheduler, BoundedCapacity = 1 });
ITargetBlock<int> target = broadcast;
var source = new TestSourceBlock<int> ();
Assert.IsTrue (broadcast.Post (1));
var header = new DataflowMessageHeader (1);
source.AddMessage (header, 2);
Assert.AreEqual (DataflowMessageStatus.Postponed,
target.OfferMessage (header, 2, source, false));
Assert.IsFalse (source.WasConsumed (header));
scheduler.ExecuteAll ();
Assert.IsTrue (source.WasConsumed (header));
}
public async Task TestReserveReleaseConsume()
{
var bb = new BroadcastBlock<int>(i => i * 2);
bb.Post(1);
await DataflowTestHelpers.TestReserveAndRelease(bb, reservationIsTargetSpecific: false);
bb = new BroadcastBlock<int>(i => i * 2);
bb.Post(2);
await DataflowTestHelpers.TestReserveAndConsume(bb, reservationIsTargetSpecific: false);
}
public void TryReceiveAllTest()
{
var scheduler = new TestScheduler();
var block = new BroadcastBlock<int>(null, new DataflowBlockOptions { TaskScheduler = scheduler });
IReceivableSourceBlock<int> source = block;
Assert.IsTrue(block.Post(1));
Assert.IsTrue(block.Post(2));
scheduler.ExecuteAll();
IList<int> items;
Assert.IsTrue(source.TryReceiveAll(out items));
CollectionAssert.AreEqual(new[] { 2 }, items);
}
private ITargetBlock <T> _targetBroadcastBlock; // reduces casting hits keeping a casted version cached
public NoLossBroadcastBlock(Func <T, T> cloningFunction)
{
_broadcastBlock = new BroadcastBlock <T>(cloningFunction);
_targetBroadcastBlock = (ITargetBlock <T>)_broadcastBlock;
_completion = _broadcastBlock.Completion;
}
public Task StartPipeline()
{
cancellationTokenSource = new CancellationTokenSource();
ExecutionDataflowBlockOptions executionDataflowBlockOptions = new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationTokenSource.Token,
//MaxDegreeOfParallelism = MAXPARA
};
broadcastSymbol = new BroadcastBlock <string>(symbol => symbol);
var joinblock = new JoinBlock <List <decimal>, List <Dividend>, KeyStats>(new GroupingDataflowBlockOptions {
Greedy = false
});
GetCompanyInfo = new TransformBlock <string, CompanyInfo>(symbol =>
{
return(RetrieveCompanyInfo(symbol));
}, executionDataflowBlockOptions);
GetDividendReports = new TransformBlock <string, List <Dividend> >(symbol =>
{
return(RetrieveDividendInfo(symbol));
}, executionDataflowBlockOptions);
GetKeyStatInfo = new TransformBlock <string, KeyStats>(symbol =>
{
return(RetrieveKeyStats(symbol));
}, executionDataflowBlockOptions);
GetIntervalReports = new TransformBlock <string, List <Interval> >(symbol =>
{
return(RetrieveIntervals(symbol, 30));
}, executionDataflowBlockOptions);
GetChangesOverInterval = new TransformBlock <List <Interval>, List <decimal> >(intervals =>
{
return(ConstructIntervalReport(intervals));
}, executionDataflowBlockOptions);
GenerateXmlString = new TransformBlock <Tuple <List <decimal>, List <Dividend>, KeyStats>, string>(tup =>
{
var ReportObj = new Report
{
changeIntervals = tup.Item1,
dividends = tup.Item2,
keyStats = tup.Item3
};
XmlSerializer ser = new XmlSerializer(typeof(Report));
var stringWriter = new StringWriter();
ser.Serialize(stringWriter, ReportObj);
return(stringWriter.ToString());
}, executionDataflowBlockOptions);
GenerateCompleteReport = new ActionBlock <string>(xml =>
{
var str = Path.GetRandomFileName().Replace(".", "") + ".xml";
File.WriteAllText(str, xml);
Console.WriteLine("Finished File");
}, executionDataflowBlockOptions);
var options = new DataflowLinkOptions {
PropagateCompletion = true
};
var buffer = new BufferBlock <string>();
buffer.LinkTo(broadcastSymbol, options);
//Broadcasts the symbol
broadcastSymbol.LinkTo(GetIntervalReports, options);
broadcastSymbol.LinkTo(GetDividendReports, options);
broadcastSymbol.LinkTo(GetKeyStatInfo, options);
//Second teir parallel
GetIntervalReports.LinkTo(GetChangesOverInterval, options);
//Joins the parallel blocks back together
GetDividendReports.LinkTo(joinblock.Target2, options);
GetKeyStatInfo.LinkTo(joinblock.Target3, options);
GetChangesOverInterval.LinkTo(joinblock.Target1, options);
joinblock.LinkTo(GenerateXmlString, options);
GenerateXmlString.LinkTo(GenerateCompleteReport, options);
buffer.Post("F");
buffer.Post("AGFS");
buffer.Post("BAC");
buffer.Post("FCF");
buffer.Complete();
broadcastSymbol.Completion.ContinueWith(tsk =>
{
if (!tsk.IsFaulted)
{
GetIntervalReports.Complete();
GetDividendReports.Complete();
GetKeyStatInfo.Complete();
}
else
{
((IDataflowBlock)GetIntervalReports).Fault(tsk.Exception);
((IDataflowBlock)GetDividendReports).Fault(tsk.Exception);
((IDataflowBlock)GetKeyStatInfo).Fault(tsk.Exception);
}
});
//TODO need to finish pipeline and find better implementation
GenerateCompleteReport.Completion.Wait(cancellationTokenSource.Token);
return(Task.CompletedTask);
}
public void RunBroadcastBlockConformanceTests()
{
bool localPassed = true;
{
// Test posting then receiving
localPassed = true;
var bb = new BroadcastBlock <int>(i => i);
for (int i = 0; i < 2; i++)
{
bb.Post(i);
}
Task.Delay(1).Wait();
localPassed |= bb.Receive() == 1;
Assert.True(localPassed, string.Format("{0}: Posting then receiving", localPassed ? "Success" : "Failure"));
}
{
// Test receiving then posting
localPassed = true;
var bb = new BroadcastBlock <int>(i => i);
Task.Run(() =>
{
Task.Delay(1).Wait();
bb.Post(42);
});
localPassed |= bb.Receive() == 42;
Assert.True(localPassed, string.Format("{0}: Receiving then posting", localPassed ? "Success" : "Failure"));
}
{
// Test broadcasting
localPassed = true;
var bb = new BroadcastBlock <int>(i => i + 1);
var tb1 = new TransformBlock <int, int>(i => i);
var tb2 = new TransformBlock <int, int>(i => i);
var tb3 = new TransformBlock <int, int>(i => i);
bb.LinkTo(tb1);
bb.LinkTo(tb2);
bb.LinkTo(tb3);
for (int i = 0; i < 2; i++)
{
bb.Post(i);
}
for (int i = 0; i < 2; i++)
{
localPassed |= tb1.Receive() == i + 1;
localPassed |= tb2.Receive() == i + 1;
localPassed |= tb3.Receive() == i + 1;
}
Assert.True(localPassed, string.Format("{0}: Broadcasting", localPassed ? "Success" : "Failure"));
}
// Test using a precanceled token
{
localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new DataflowBlockOptions {
CancellationToken = cts.Token
};
var bb = new BroadcastBlock <int>(i => i, dbo);
int ignoredValue;
IList <int> ignoredValues;
localPassed &= bb.LinkTo(new ActionBlock <int>(delegate { })) != null;
localPassed &= bb.SendAsync(42).Result == false;
localPassed &= ((IReceivableSourceBlock <int>)bb).TryReceiveAll(out ignoredValues) == false;
localPassed &= bb.Post(42) == false;
localPassed &= bb.TryReceive(out ignoredValue) == false;
localPassed &= bb.Completion != null;
bb.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, string.Format("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"));
}
}
public void PostponedTest ()
{
var broadcast = new BroadcastBlock<int> (null);
var target = new BufferBlock<int> (
new DataflowBlockOptions { BoundedCapacity = 1 });
broadcast.LinkTo (target);
Assert.IsTrue (target.Post (1));
Assert.IsTrue (broadcast.Post (2));
Assert.AreEqual (1, target.Receive (TimeSpan.FromMilliseconds (0)));
Assert.AreEqual (2, target.Receive (TimeSpan.FromMilliseconds (100)));
}
static void Main(string[] args)
{
var broadcastBlock =
new BroadcastBlock <int>(a => a,
new DataflowBlockOptions()
{
BoundedCapacity = 1
}
);
var a1 = new TransformBlock <int, int>(
a =>
{
Console.WriteLine($"Message {a} was processed by consumer 1");
if (a % 2 == 0)
{
Task.Delay(300).Wait();
}
else
{
Task.Delay(50).Wait();
}
return(a * -1);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 3
});
var a2 = new TransformBlock <int, int>(
a =>
{
Console.WriteLine($"Message {a} was processed by consumer 2");
if (a % 2 != 0)
{
Task.Delay(300).Wait();
}
else
{
Task.Delay(50).Wait();
}
return(a);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 3
});
broadcastBlock.LinkTo(a1);
broadcastBlock.LinkTo(a2);
var joinblock = new JoinBlock <int, int>();
a1.LinkTo(joinblock.Target1);
a2.LinkTo(joinblock.Target2);
var printBlock = new ActionBlock <Tuple <int, int> >(
a => Console.WriteLine($"Message {a} was processed. Sum: {a.Item1 + a.Item2}")
);
joinblock.LinkTo(printBlock);
for (int i = 0; i < 10; i++)
{
var i1 = i;
broadcastBlock.SendAsync(i)
.ContinueWith(a =>
{
Console.WriteLine(a.Result ? $"Messgae {i1} was accepted" : $"Messgae {i1} was rejected");
});
}
Console.WriteLine("done");
Console.ReadLine();
}
public override void ReadFrames(BroadcastBlock <FrameData> broadcast)
{
_broadcast = broadcast;
}
public DiagnosticPipeline(
IHealthReporter healthReporter,
IReadOnlyCollection <IObservable <EventData> > inputs,
IReadOnlyCollection <IFilter> globalFilters,
IReadOnlyCollection <EventSink> sinks,
DiagnosticPipelineConfiguration pipelineConfiguration = null,
bool disposeDependencies = false,
TaskScheduler taskScheduler = null)
{
Requires.NotNull(healthReporter, nameof(healthReporter));
Requires.NotNull(inputs, nameof(inputs));
Requires.Argument(inputs.Count > 0, nameof(inputs), "There must be at least one input");
Requires.NotNull(sinks, nameof(sinks));
Requires.Argument(sinks.Count > 0, nameof(sinks), "There must be at least one sink");
this.batcherTimerDisposalLock = new object();
this.pipelineConfiguration = pipelineConfiguration ?? new DiagnosticPipelineConfiguration();
taskScheduler = taskScheduler ?? TaskScheduler.Current;
// An estimatie how many batches of events to allow inside the pipeline.
// We want to be able to process full buffer of events, but also have enough batches in play in case of high concurrency.
int MaxNumberOfBatchesInProgress = Math.Max(
5 * this.pipelineConfiguration.MaxConcurrency,
this.pipelineConfiguration.PipelineBufferSize / this.pipelineConfiguration.MaxEventBatchSize);
this.Inputs = inputs;
this.Sinks = sinks;
// Just play nice and make sure there is always something to enumerate on
this.GlobalFilters = globalFilters ?? new IFilter[0];
this.HealthReporter = healthReporter;
this.cancellationTokenSource = new CancellationTokenSource();
var propagateCompletion = new DataflowLinkOptions()
{
PropagateCompletion = true
};
// One disposable for each input subscription.
this.inputSubscriptions = new List <IDisposable>(inputs.Count);
var inputBuffer = new BufferBlock <EventData>(
new DataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.PipelineBufferSize,
CancellationToken = this.cancellationTokenSource.Token,
TaskScheduler = taskScheduler
});
this.pipelineHead = inputBuffer;
var batcher = new BatchBlock <EventData>(
this.pipelineConfiguration.MaxEventBatchSize,
new GroupingDataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.PipelineBufferSize,
CancellationToken = this.cancellationTokenSource.Token,
TaskScheduler = taskScheduler
}
);
inputBuffer.LinkTo(batcher, propagateCompletion);
ISourceBlock <EventData[]> sinkSource;
FilterAction filterTransform;
if (this.GlobalFilters.Count > 0)
{
filterTransform = new FilterAction(
this.GlobalFilters,
this.cancellationTokenSource.Token,
MaxNumberOfBatchesInProgress,
this.pipelineConfiguration.MaxConcurrency,
healthReporter,
taskScheduler);
var globalFiltersBlock = filterTransform.GetFilterBlock();
batcher.LinkTo(globalFiltersBlock, propagateCompletion);
sinkSource = globalFiltersBlock;
}
else
{
sinkSource = batcher;
}
bool usingBroadcastBlock = sinks.Count > 1;
if (usingBroadcastBlock)
{
var broadcaster = new BroadcastBlock <EventData[]>(
(events) => events?.Select((e) => e.DeepClone()).ToArray(),
new DataflowBlockOptions()
{
BoundedCapacity = MaxNumberOfBatchesInProgress,
CancellationToken = this.cancellationTokenSource.Token,
TaskScheduler = taskScheduler
});
sinkSource.LinkTo(broadcaster, propagateCompletion);
sinkSource = broadcaster;
}
this.outputCompletionTasks = new List <Task>(sinks.Count);
foreach (var sink in sinks)
{
ISourceBlock <EventData[]> outputSource = sinkSource;
if (sink.Filters != null && sink.Filters.Count > 0)
{
filterTransform = new FilterAction(
sink.Filters,
this.cancellationTokenSource.Token,
MaxNumberOfBatchesInProgress,
this.pipelineConfiguration.MaxConcurrency,
healthReporter,
taskScheduler);
var filterBlock = filterTransform.GetFilterBlock();
if (usingBroadcastBlock)
{
var lossReportingPropagator = new LossReportingPropagatorBlock <EventData[]>(this.HealthReporter);
sinkSource.LinkTo(lossReportingPropagator, propagateCompletion);
lossReportingPropagator.LinkTo(filterBlock, propagateCompletion);
}
else
{
sinkSource.LinkTo(filterBlock, propagateCompletion);
}
outputSource = filterBlock;
}
else if (usingBroadcastBlock)
{
var lossReportingPropagator = new LossReportingPropagatorBlock <EventData[]>(this.HealthReporter);
sinkSource.LinkTo(lossReportingPropagator, propagateCompletion);
outputSource = lossReportingPropagator;
}
OutputAction outputAction = new OutputAction(
sink.Output,
this.cancellationTokenSource.Token,
MaxNumberOfBatchesInProgress,
this.pipelineConfiguration.MaxConcurrency,
healthReporter,
taskScheduler);
var outputBlock = outputAction.GetOutputBlock();
outputSource.LinkTo(outputBlock, propagateCompletion);
this.outputCompletionTasks.Add(outputBlock.Completion);
}
IObserver <EventData> inputBufferObserver = new TargetBlockObserver <EventData>(inputBuffer, this.HealthReporter);
foreach (var input in inputs)
{
this.inputSubscriptions.Add(input.Subscribe(inputBufferObserver));
}
this.disposed = false;
this.disposeDependencies = disposeDependencies;
this.batcherTimer = new Timer(
(_) => {
try
{
lock (this.batcherTimerDisposalLock)
{
if (!this.disposed)
{
batcher.TriggerBatch();
this.batcherTimer.Change(dueTime: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec), period: Timeout.InfiniteTimeSpan);
}
}
}
catch { }
},
state: null,
dueTime: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec),
period: Timeout.InfiniteTimeSpan);
}
