public void TestTransformManyBlockConstructor() { // IEnumerable without option var block = new TransformManyBlock<int, string>(i => new string[10]); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock."); // Task without option block = new TransformManyBlock<int, string>(i => Task.Factory.StartNew(() => (IEnumerable<string>)new string[10])); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock."); // IEnumerable with not cancelled token and default scheduler block = new TransformManyBlock<int, string>(i => new string[10], new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock."); // Task with not cancelled token and default scheduler block = new TransformManyBlock<int, string>(i => Task.Factory.StartNew(() => (IEnumerable<string>)new string[10]), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock."); // IEnumerable with a cancelled token and default scheduler var token = new CancellationToken(true); block = new TransformManyBlock<int, string>(i => new string[10], new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock."); // Task with a cancelled token and default scheduler token = new CancellationToken(true); block = new TransformManyBlock<int, string>(i => Task.Factory.StartNew(() => (IEnumerable<string>)new string[10]), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock."); }
internal EvalOperation(OnlineTrainerSettingsInternal settings) { this.telemetry = new TelemetryClient(); // evaluation pipeline this.evalEventHubClient = EventHubClient.CreateFromConnectionString(settings.EvalEventHubConnectionString); this.evalBlock = new TransformManyBlock<object, EvalData>( (Func<object, IEnumerable<EvalData>>)this.OfflineEvaluate, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4, BoundedCapacity = 1024 }); this.evalBlock.Completion.ContinueWith(t => { this.telemetry.TrackTrace($"Stage 3 - Evaluation pipeline completed: {t.Status}"); if (t.IsFaulted) this.telemetry.TrackException(t.Exception); }); // batch output together to match EventHub throughput by maintaining maximum latency of 5 seconds this.evalBlockDisposable = this.evalBlock.AsObservable() .GroupBy(k => k.PolicyName) .Select(g => g.Window(TimeSpan.FromSeconds(5)) .Select(w => w.Buffer(245 * 1024, e => Encoding.UTF8.GetByteCount(e.JSON))) .SelectMany(w => w) .Subscribe(this.UploadEvaluation)) .Publish() .Connect(); }
static public void ProcessingByTPL_StraightForwardImplementation() { const string pathToFiles = @"..\..\..\..\DataFiles"; string[] files = Directory.GetFiles(pathToFiles, "*.txt", SearchOption.AllDirectories); var loadDataFromFileBlock = new TransformBlock<string[], List<CustomerTextData>>(fileItems => { var factory = new CustomerTextDataFactory(); return new List<CustomerTextData>(Array.ConvertAll(fileItems, factory.LoadFromFile)); }); var filterBlock = new TransformBlock<List<CustomerTextData>, List<CustomerTextData>>(textDataList => { var filter = new FilterTextData(5); return textDataList.Where(filter.Run).ToList(); }); var toListBlock = new TransformManyBlock<List<CustomerTextData>, CustomerTextData>(textDataList => { var queue = new ConcurrentQueue<CustomerTextData>(); textDataList.ForEach(queue.Enqueue); return queue; }); var action = new ActionBlock<CustomerTextData>(textData => { var weight = new WeightTextData(); int result = weight.Run(textData); Trace.WriteLine(result); Console.WriteLine(result); }); loadDataFromFileBlock.LinkTo(filterBlock); filterBlock.LinkTo(toListBlock); toListBlock.LinkTo(action); loadDataFromFileBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)filterBlock).Fault(t.Exception); else filterBlock.Complete(); }); filterBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)toListBlock).Fault(t.Exception); else toListBlock.Complete(); }); toListBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)action).Fault(t.Exception); else action.Complete(); }); loadDataFromFileBlock.Post(files); loadDataFromFileBlock.Complete(); action.Completion.Wait(); }
private static TransformManyBlock<int, int> ConstructTransformManyWithNMessages(int messagesCount) { var block = new TransformManyBlock<int, int>(i => new int[] { i }); for (int i = 0; i < messagesCount; i++) { block.Post(i); } // Spin until the messages have been properly buffered up. // Otherwise TryReceive fails. SpinWait.SpinUntil(() => block.OutputCount == messagesCount); return block; }
public void NullResultTest () { bool received = false; var transformMany = new TransformManyBlock<int, int> (i => (IEnumerable<int>)null); var action = new ActionBlock<int> (i => received = true); transformMany.LinkTo (action); Assert.IsTrue (transformMany.Post (1)); transformMany.Complete (); Assert.IsTrue (transformMany.Completion.Wait (100)); Assert.IsFalse (received); }
public PlayerMatchProducer(RiotApi riotApi, RiotQuerySettings querySettings, List<Queue> queryQueues, MatchDownloadLimiter testSynchronizer) { api = riotApi; this.querySettings = querySettings; this.queryQueues = queryQueues; this.testSynchronizer = testSynchronizer; // Create blocks PlayerBufferBlock = new BufferBlock<PlayerEntry>(); PlayerToMatchesBlock = new TransformManyBlock<PlayerEntry, MatchSummary>( async player => await ConsumePlayerAsync(player), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 2 }); // Link blocks PlayerBufferBlock.LinkTo(PlayerToMatchesBlock, new DataflowLinkOptions() { PropagateCompletion = true }); }
public async Task TestAutoComplete2() { var block2 = new TransformManyBlock<Int, Int>(i => { return Enumerable.Empty<Int>(); }); var dataflow1 = DataflowUtils.FromDelegate<Int, Int>(i => new[] { i }); //preserve the guid var dataflow2 = DataflowUtils.FromBlock(block2).AutoComplete(TimeSpan.FromSeconds(1)); dataflow1.GoTo(dataflow2).GoTo(dataflow1); dataflow1.InputBlock.Post(new Int() { Val = 1 }); Assert.IsTrue(await dataflow2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(2))); Assert.IsTrue(dataflow2.Name.EndsWith("AutoComplete")); }
public void DeferredUsageTest () { int insIndex = -1; int[] array = new int[5 + 3]; var evt = new CountdownEvent (array.Length); var block = new ActionBlock<int> (i => { array[Interlocked.Increment (ref insIndex)] = i; evt.Signal (); }); var trsm = new TransformManyBlock<int, int> (i => Enumerable.Range (0, i)); trsm.Post (5); trsm.Post (3); trsm.LinkTo (block); evt.Wait (); CollectionAssert.AreEquivalent (new int[] { 0, 1, 2, 3, 4, 0, 1, 2 }, array); }
private static IPropagatorBlock<string, KeyValuePair<string, long>> CreateCustomBlock() { var directoryBrowserBlock = new TransformManyBlock<string, string>(path => { var dir = new DirectoryInfo(path); return dir.EnumerateFileSystemInfos() .Select(fi => fi.FullName); }); var fileSizeCalculator = new TransformBlock<string, KeyValuePair<string, long>>(fileName => { var fi = new FileInfo(fileName); return new KeyValuePair<string, long>(fileName, fi.Length); }); directoryBrowserBlock.LinkTo(fileSizeCalculator, new DataflowLinkOptions { PropagateCompletion = true }, File.Exists); var customBlock = DataflowBlock.Encapsulate(directoryBrowserBlock, fileSizeCalculator); return customBlock; }
public async Task TestBlockBufferCount() { var block1 = new TransformBlock<int, int>(i => 2 * i); var block2 = new TransformManyBlock<int, int>(i => new [] { i }); var block3 = new ActionBlock<int>(i => { Thread.Sleep(1000); }); block1.Post(0); block2.Post(0); block2.Post(0); block3.Post(0); block3.Post(0); block3.Post(0); await Task.Delay(200); Assert.AreEqual(1, block1.GetBufferCount().Total()); Assert.AreEqual(2, block2.GetBufferCount().Total()); Assert.AreEqual(2, block3.GetBufferCount().Total()); }
public async Task TestAutoComplete1() { var block = new TransformManyBlock<Int, Int>(i => { int j = i.Val + 1; Console.WriteLine("block2: i = {0}, j = {1}", i.Val, j); Thread.Sleep(500); if (j < 10) return new[] { new Int { Val = j } }; else return Enumerable.Empty<Int>(); }); var dataflow1 = DataflowUtils.FromDelegate<Int, Int>(i => i); var dataflow2 = DataflowUtils.FromBlock(block).AutoComplete(TimeSpan.FromSeconds(1)); dataflow1.GoTo(dataflow2).GoTo(dataflow1); dataflow1.InputBlock.Post(new Int() { Val = 1 }); Assert.IsTrue(await dataflow2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(10))); }
public async static Task<Animat> SeedAsyncWithout3MB(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry) { var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest); var result = new Animat { ScenarioSpecies = species }; var area = bounds.Area; //Debug.WriteLine("Area: {0}",area); var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count => { var geos = new List<Geo<float>>(); for (var i = 0; i < count; i++) { var location = bounds.RandomLocationWithinPerimeter(); var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data; if (depth < -50) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble()))); } return geos; }, new ExecutionDataflowBlockOptions { TaskScheduler = TaskScheduler.Default, BoundedCapacity = -1, MaxDegreeOfParallelism = -1, }); var bufferBlock = new BufferBlock<Geo<float>>(); transformManyBlock.LinkTo(bufferBlock); var population = (int)Math.Round(area * species.PopulationDensity); result.TotalAnimats = population; const int blockSize = 100; while (population > 0) { transformManyBlock.Post(population > blockSize ? blockSize : population); population -= blockSize; } transformManyBlock.Complete(); await transformManyBlock.Completion; IList<Geo<float>> animatGeos; if (bufferBlock.TryReceiveAll(out animatGeos)) result.Locations.AddRange(animatGeos); return result; }
public BreakAvailabilityCalculator( ILogger logger, ISqlServerDbContextFactory <ISqlServerLongRunningTenantDbContext> tenantDbContextFactory, RecalculateBreakAvailabilityOptions recalculateOptions, ITargetBlock <IBreakAvailability> targetBlock, CancellationToken cancellationToken) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _tenantDbContextFactory = tenantDbContextFactory ?? throw new ArgumentNullException(nameof(tenantDbContextFactory)); _recalculateOptions = recalculateOptions ?? throw new ArgumentNullException(nameof(recalculateOptions)); _cancellationToken = cancellationToken; _internalBlock = new TransformManyBlock <IBreakAvailability[], IBreakAvailability>( breaks => breaks.AsEnumerable(), new ExecutionDataflowBlockOptions { CancellationToken = _cancellationToken }); _ = _internalBlock.LinkTo(targetBlock ?? throw new ArgumentNullException(nameof(targetBlock))); _ = _internalBlock.Completion.ContinueWith(_ => targetBlock.Complete()); _lockDbContext = new SemaphoreSlim(_recalculateOptions.BoundedDbContextOperationCapacity, _recalculateOptions.BoundedDbContextOperationCapacity); }
public async Task TestAutoComplete3() { var block1 = new TransformManyBlock <Int, Int>(i => { return(new[] { i }); //preserve the guid }); var block2 = new TransformManyBlock <Int, Int>(i => { return(Enumerable.Empty <Int>()); }); var container1 = BlockContainerUtils.FromBlock(block1); var container2 = BlockContainerUtils.AutoComplete(BlockContainerUtils.FromBlock(block2), TimeSpan.FromSeconds(1)); container1.LinkTo(container2); container2.LinkTo(container1); container1.InputBlock.Post(new Int() { Val = 1 }); Assert.IsTrue(await container2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(2))); }
public void TransformManyOverfullTest() { var scheduler = new TestScheduler(); int n = 0; var transform = new TransformManyBlock <int, int> ( i => { Interlocked.Increment(ref n); return(new[] { -i, i }); }, new ExecutionDataflowBlockOptions { BoundedCapacity = 2, TaskScheduler = scheduler }); Assert.IsTrue(transform.Post(1)); Assert.IsTrue(transform.Post(2)); Assert.GreaterOrEqual(scheduler.ExecuteAll(), 1); Assert.AreEqual(2, Volatile.Read(ref n)); }
public LoopDataflow2() { var options = new ExecutionDataflowBlockOptions() { BoundedCapacity = 100 }; InputMessageBlock = new TransformBlock <Message, Message>(async msg => await InputMessage(msg)); HandleMessageBlock = new TransformManyBlock <Message, Message>(async msg => await HandleMessage(msg), options); HandleMessageBlock2 = new TransformManyBlock <Message, Message>(async msg => await HandleMessage2(msg), options); OutputMessageBlock = new ActionBlock <Message>(msg => OutputMessage(msg), options); var linkOptions = new DataflowLinkOptions() { PropagateCompletion = false }; InputMessageBlock.LinkTo(HandleMessageBlock, linkOptions); HandleMessageBlock.LinkTo(OutputMessageBlock, linkOptions, msg => msg.WasProcessed == true); HandleMessageBlock.LinkTo(HandleMessageBlock2, linkOptions, msg => msg.WasProcessed == false); HandleMessageBlock2.LinkTo(OutputMessageBlock, linkOptions, msg => msg.WasProcessed == true); HandleMessageBlock2.LinkTo(HandleMessageBlock, linkOptions, msg => msg.WasProcessed == false); InputMessageBlock.Completion.ContinueWith(async tsk => { await BothMessageHandlersAreComplete(); HandleMessageBlock.Complete(); }); HandleMessageBlock.Completion.ContinueWith(tsk => { HandleMessageBlock2.Complete(); }); HandleMessageBlock2.Completion.ContinueWith(tsk => { OutputMessageBlock.Complete(); }); DebuggingLoop(); }
/// <summary> /// 下载文件方法(自动确定是否使用分片下载) /// </summary> /// <param name="fileEnumerable">文件列表</param> /// <param name="downloadParts"></param> public static async Task AdvancedDownloadListFile(IEnumerable <DownloadFile> fileEnumerable, int downloadParts = 16) { ProcessorHelper.SetMaxThreads(); var filesBlock = new TransformManyBlock <IEnumerable <DownloadFile>, DownloadFile>(d => { var dl = d.ToList(); foreach (var df in dl.Where(df => !Directory.Exists(df.DownloadPath))) { Directory.CreateDirectory(df.DownloadPath); } return(dl); }); var actionBlock = new ActionBlock <DownloadFile>(async d => { if (d.FileSize is >= 1048576 or 0) { await MultiPartDownloadTaskAsync(d, downloadParts); }
public async Task TestAutoComplete2() { var block1 = new TransformManyBlock <Int, Int>(i => { return(new[] { i }); //preserve the guid }); var block2 = new TransformManyBlock <Int, Int>(i => { int j = i.Val + 1; Console.WriteLine("block2: i = {0}, j = {1}", i.Val, j); Thread.Sleep(500); if (j < 10) { return new[] { new Int { Val = j } } } ; else { return(Enumerable.Empty <Int>()); } }); var container1 = BlockContainerUtils.FromBlock(block1); var container2 = BlockContainerUtils.AutoComplete(BlockContainerUtils.FromBlock(block2), TimeSpan.FromSeconds(1)); container1.LinkTo(container2); container2.LinkTo(container1); container1.InputBlock.Post(new Int() { Val = 1 }); Assert.IsTrue(await container2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(10))); }
// TransformManyBlock is opposite to a BatchBlock. It takes a single input and creates many outputs. // We will also be demonstrating how to link to an ActionBlock to work on those outputs. // Also make note that the order is still ensured. private static async Task SimpleDemoWithParallelismAsync() { Console.WriteLine("TransformManyBlockDemo has started!"); var transformManyBlock = new TransformManyBlock <int, string>( ProduceTimeData, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 }); var actionBlock = new ActionBlock <string>(Console.WriteLine); transformManyBlock.Post(20); // Connect the source block (Transform) to the target/action block (Action). transformManyBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true }); transformManyBlock.Complete(); await transformManyBlock.Completion.ConfigureAwait(false); Console.WriteLine("Finished!"); Console.ReadKey(); }
private ISourceBlock <RunningGame> GameProcessPoller(Config config, IEnumerable <IGameProcessInfo> gameProcessInfos) { Dictionary <string, string> gameByProcess = GameByProcess(gameProcessInfos); Func <IEnumerable <string>, IEnumerable <string?> > keepOne = new KeepOne <string>(gameByProcess.Keys).Call; var transformer = new TransformManyBlock <IEnumerable <string>, RunningGame>( processNames => keepOne(processNames).Select(processName => { if (processName == null) { _logger.LogInformation("The game has been closed"); return(new RunningGame(null)); } var game = gameByProcess.GetValueOrDefault(processName); _logger.LogInformation("Found game {string?} (process {string?})", game, processName); return(new RunningGame(game)); }) ); var source = PollingSource.Create(config.PollingInterval, () => Process.GetProcesses().Select(p => p.ProcessName)); source.LinkTo(transformer); transformer.Completion.ContinueWith(_ => source.Complete()); return(transformer); }
/// <summary> /// The following basic example creates a TransformManyBlock<TInput,TOutput> object /// that splits strings into their individual character sequences. The TransformManyBlock<TInput,TOutput> /// object takes String values as input and produces Char values as output. /// </summary> /// <param name="type"></param> public static void Run(int type = 1) { // Create a TransformManyBlock<string, char> object that splits // a string into its individual characters. TransformManyBlock <string, char> transformManyBlock = new TransformManyBlock <string, char>(s => { char[] charArray = s.ToCharArray(); return(charArray); }); // Post two messages to the first block. transformManyBlock.Post("Hello"); transformManyBlock.Post("World"); // Receive all output values from the block. if (type == 0) { for (int i = 0; i < ("Hello" + "World").Length; i++) { Console.WriteLine(transformManyBlock.Receive()); } } else if (type == 1) { do { if (transformManyBlock.InputCount == 0 && transformManyBlock.OutputCount == 0) { return; } char receive = transformManyBlock.Receive(); Console.WriteLine("transformManyBlock.Receive:"); Console.WriteLine(receive); Console.WriteLine(); } while (true); } }
public async Task TestCountZeroAtCompletion() { var cts = new CancellationTokenSource(); var tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token }); tb.Post(1); cts.Cancel(); await Assert.ThrowsAnyAsync <OperationCanceledException>(() => tb.Completion); Assert.Equal(expected: 0, actual: tb.InputCount); Assert.Equal(expected: 0, actual: tb.OutputCount); cts = new CancellationTokenSource(); tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable); tb.Post(1); ((IDataflowBlock)tb).Fault(new InvalidOperationException()); await Assert.ThrowsAnyAsync <InvalidOperationException>(() => tb.Completion); Assert.Equal(expected: 0, actual: tb.InputCount); Assert.Equal(expected: 0, actual: tb.OutputCount); }
public async Task TestCircularLinkingAsyncEnumerable() { const int Iters = 200; var tcs = new TaskCompletionSource <bool>(); IAsyncEnumerable <int> body(int i) { if (i >= Iters) { tcs.SetResult(true); } return(AsyncEnumerable.Repeat(i + 1, 1)); } TransformManyBlock <int, int> tb = new TransformManyBlock <int, int>(body); using (tb.LinkTo(tb)) { tb.Post(0); await tcs.Task; tb.Complete(); } }
private static void Main(string[] args) { var generatorBlock = new TransformManyBlock<int, string>(num => GenerateStrings(num)); var writerBlock = new TransformBlock<string, string>(str => WriteString(str), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 }); var finishingBlock = new ActionBlock<string>(str => { writerBlock.Completion.Wait(); Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ": finished - " + str); }); generatorBlock.LinkTo(writerBlock, new DataflowLinkOptions{PropagateCompletion = true}); writerBlock.LinkTo(finishingBlock, new DataflowLinkOptions { PropagateCompletion = true }); for (var i = 1; i <= 3; i++) { Console.WriteLine("Posted " + i*10); generatorBlock.Post(i*10); } generatorBlock.Complete(); finishingBlock.Completion.Wait(); Console.WriteLine("Pipeline is finished"); Console.ReadKey(); }
public async Task TestYieldingNoResults() { foreach (int dop in new[] { 1, Environment.ProcessorCount }) foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 1, 2 }) { const int Modes = 3, Iters = 100; var tb = new TransformManyBlock<int, int>(i => { switch (i % Modes) { default: case 0: return new List<int> { i }; case 1: return new int[0]; case 2: return new Collection<int> { i, i + 1 }; } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = boundedCapacity }); var source = new BufferBlock<int>(); source.PostRange(0, Modes * Iters); source.Complete(); source.LinkTo(tb, new DataflowLinkOptions { PropagateCompletion = true }); int received = 0; while (await tb.OutputAvailableAsync()) { await tb.ReceiveAsync(); received++; } Assert.Equal(expected: Modes * Iters, actual: received); } }
private static IDictionary <string, uint> GetTopWordsDataFlow() { Console.WriteLine(nameof(GetTopWordsDataFlow) + "..."); const int WorkerCount = 12; var result = new ConcurrentDictionary <string, uint>(StringComparer.InvariantCultureIgnoreCase); const int BoundedCapacity = 10000; var bufferBlock = new BufferBlock <string>( new DataflowBlockOptions { BoundedCapacity = BoundedCapacity }); var splitLineToWordsBlock = new TransformManyBlock <string, string>( line => line.Split(Separators, StringSplitOptions.RemoveEmptyEntries), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, BoundedCapacity = BoundedCapacity }); var batchWordsBlock = new BatchBlock <string>(5000); var trackWordsOccurrencBlock = new ActionBlock <string[]>(words => { foreach (var word in words) { if (!IsValidWord(word)) { continue; } result.AddOrUpdate(word, 1, (key, oldVal) => oldVal + 1); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = WorkerCount }); var defaultLinkOptions = new DataflowLinkOptions { PropagateCompletion = true }; bufferBlock.LinkTo(splitLineToWordsBlock, defaultLinkOptions); splitLineToWordsBlock.LinkTo(batchWordsBlock, defaultLinkOptions); batchWordsBlock.LinkTo(trackWordsOccurrencBlock, defaultLinkOptions); // Begin producing foreach (var line in File.ReadLines(InputFile.FullName)) { bufferBlock.SendAsync(line).Wait(); } bufferBlock.Complete(); // End of producing // Wait for workers to finish their work trackWordsOccurrencBlock.Completion.Wait(); return(result .OrderByDescending(kv => kv.Value) .Take((int)TopCount) .ToDictionary(kv => kv.Key, kv => kv.Value)); }
public void AsyncNullTest () { var scheduler = new TestScheduler (); var block = new TransformManyBlock<int, int> ( i => (Task<IEnumerable<int>>)null, new ExecutionDataflowBlockOptions { TaskScheduler = scheduler }); Assert.IsTrue (block.Post (1)); scheduler.ExecuteAll (); Assert.IsFalse (block.Completion.Wait (100)); block.Complete (); Assert.IsTrue (block.Completion.Wait (100)); }
private Tuple <ITargetBlock <Assembly>, Task <DiscoveredParts> > CreateAssemblyDiscoveryBlockChain(IProgress <DiscoveryProgress>?progress, CancellationToken cancellationToken) { var progressFilter = new ProgressFilter(progress); var tuple = this.CreateDiscoveryBlockChain(false, progressFilter, cancellationToken); var exceptions = new List <PartDiscoveryException>(); var assemblyBlock = new TransformManyBlock <Assembly, Type>( a => { IReadOnlyCollection <Type> types; try { // Fully realize any enumerable now so that we can catch the exception rather than // leave it to dataflow to catch it. types = this.GetTypes(a).ToList(); } catch (ReflectionTypeLoadException ex) { var partDiscoveryException = new PartDiscoveryException(string.Format(CultureInfo.CurrentCulture, Strings.ReflectionTypeLoadExceptionWhileEnumeratingTypes, a.Location), ex) { AssemblyPath = a.Location }; lock (exceptions) { exceptions.Add(partDiscoveryException); } types = ex.Types.Where(t => t != null).ToList(); } catch (Exception ex) { var partDiscoveryException = new PartDiscoveryException(string.Format(CultureInfo.CurrentCulture, Strings.UnableToEnumerateTypes, a.Location), ex) { AssemblyPath = a.Location }; lock (exceptions) { exceptions.Add(partDiscoveryException); } return(Enumerable.Empty <Type>()); } progressFilter.OnDiscoveredMoreTypes(types.Count); return(types); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Debugger.IsAttached ? 1 : Environment.ProcessorCount, CancellationToken = cancellationToken, }); assemblyBlock.LinkTo(tuple.Item1, new DataflowLinkOptions { PropagateCompletion = true }); var tcs = new TaskCompletionSource <DiscoveredParts>(); Task.Run(async delegate { try { var parts = await tuple.Item2.ConfigureAwait(false); tcs.SetResult(parts.Merge(new DiscoveredParts(Enumerable.Empty <ComposablePartDefinition>(), exceptions))); } catch (Exception ex) { tcs.SetException(ex); } }); return(Tuple.Create <ITargetBlock <Assembly>, Task <DiscoveredParts> >(assemblyBlock, tcs.Task)); }
static void Main() { var sw = new Stopwatch(); sw.Start(); var loadSourceFileToMemory = new TransformBlock <string, string>(async path => { Console.WriteLine("Loading to memory '{0}'...", path); using (StreamReader SourceReader = File.OpenText(path)) { return(await SourceReader.ReadToEndAsync()); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxDegreeOfParallelismLoad }); var generateTestClass = new TransformManyBlock <string, TestClass>(async text => { Console.WriteLine("Generating test classes..."); TestClassGenerator classGenerator = new TestClassGenerator(); return(classGenerator.GenerateTestClasses(text)); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxDegreeOfParallelismGenerate }); var saveTestClassToFile = new ActionBlock <TestClass>(async testClass => { using (StreamWriter DestinationWriter = File.CreateText(DestPath + testClass.FileName)) { Console.WriteLine("Saving '{0}' on disk...", testClass.FileName); await DestinationWriter.WriteAsync(testClass.Source); Console.WriteLine("Saved '{0}'!", testClass.FileName); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxDegreeOfParallelismSave }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; loadSourceFileToMemory.LinkTo(generateTestClass, linkOptions); generateTestClass.LinkTo(saveTestClassToFile, linkOptions); foreach (var path in _input) { loadSourceFileToMemory.Post(path); } loadSourceFileToMemory.Complete(); saveTestClassToFile.Completion.Wait(); sw.Stop(); Console.WriteLine(sw.ElapsedMilliseconds); }
public void Test() { var action = new ActionBlock <int[]>(x => { }, new ExecutionDataflowBlockOptions { }); var batch = new BatchBlock <int>(10, new GroupingDataflowBlockOptions { }); using var likBA = batch.LinkTo(action, new DataflowLinkOptions { }, x => x[0] > 1); //actionBlock gets 10 elements batch.TriggerBatch(); //for leftover elements var batchJoined = new BatchedJoinBlock <int[], string>(10, new GroupingDataflowBlockOptions { }); batch.LinkTo(batchJoined.Target1, new DataflowLinkOptions { }, ints => ints[0] > 0); var action2 = new ActionBlock <Tuple <IList <int[]>, IList <string> > >(x => { }); using var linkBJA = batchJoined.LinkTo(action2, new DataflowLinkOptions { }, x => x.Item1.Count == x.Item2.Count);//it's 10 total so 7-3 or 0-10 will propagate var broadcast = new BroadcastBlock <int>(cloningFunction: x => x, new DataflowBlockOptions { }); var action3 = new ActionBlock <int>(x => { }); var action4 = new ActionBlock <int>(x => { }); broadcast.LinkTo(action3, new DataflowLinkOptions { }, x => x > 0); broadcast.LinkTo(action4, new DataflowLinkOptions { }, x => x > 0); //both will gete same elements var buffer = new BufferBlock <int>(new DataflowBlockOptions { }); var action5 = new ActionBlock <int>(x => { }); buffer.LinkTo(action5, new DataflowLinkOptions { }, x => x == 0); var buffer1 = new BufferBlock <int>(); var join = new JoinBlock <int, string>(new GroupingDataflowBlockOptions { }); buffer1.LinkTo(join.Target1); var action6 = new ActionBlock <Tuple <int, string> >(tlp => { }); join.LinkTo(action6); var transform = new TransformBlock <int, string>(x => x.ToString(), new ExecutionDataflowBlockOptions { }); var action7 = new ActionBlock <string>(str => { }); transform.LinkTo(action7); var transformMany = new TransformManyBlock <int, string>( x => Enumerable.Range(0, x).Select(i => i.ToString()));//one recived to many output var action8 = new ActionBlock <string>(str => { }); var writeOnce = new WriteOnceBlock <int>(cloningFunction: x => x, new DataflowBlockOptions { }); // it gets and stores 1 element but gives clones to linked blocks? var action9 = new ActionBlock <int>(x => { }); writeOnce.LinkTo(action9); var bufferC1 = new BufferBlock <int>(); var bufferC2 = new BufferBlock <string>(); var actionIdx = DataflowBlock.Choose <int, string>( bufferC1, x => Console.WriteLine(x + 1), bufferC2, str => Console.WriteLine(str + "")); var actionE1 = new ActionBlock <int>(x => { }); var bufferE1 = new BufferBlock <string>(); var propagator = DataflowBlock.Encapsulate <int, string>(actionE1, bufferE1); var scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, Environment.ProcessorCount, int.MaxValue); var concurrent = scheduler.ConcurrentScheduler; var exclusive = scheduler.ExclusiveScheduler; }
public void BuildPipeline(CancellationToken token) { // create the blocks getFileUrls = new TransformManyBlock<string, string>( (url) => { List<string> fileUrls = new List<string>(); GetFileUrls(url, fileUrls); return fileUrls; } , new ExecutionDataflowBlockOptions { CancellationToken = token } ); var downloadFile = new TransformBlock<string, string>( (fileUrl) => { string fileName = Regex.Match(fileUrl, @"([^/]+$)").Value; using (var client = new WebClient()) { client.DownloadFile(fileUrl, FolderPath + "\\" + fileName); } Interlocked.Increment(ref currentFileCount); UpdateProgress((double)currentFileCount / (double)fileCount); return FolderPath + "\\" + fileName; } , new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = token, } ); var convertFile = new TransformBlock<string, string>( (fileName) => { string newFileName = ConvertedFilesFolder + @"\" + Regex.Match(fileName, @"([^\\]+$)").Value + "x"; Process process = new Process(); process.StartInfo.Arguments = string.Format(@" -nme -oice {0} {1}", fileName, newFileName); process.StartInfo.FileName = @"c:\Program Files (x86)\Microsoft Office\Office12\excelcnv.exe"; process.Start(); while (!process.WaitForExit(15000)) { process.Kill(); process.Start(); } Interlocked.Increment(ref convertedFilesCount); UpdateConversionProgress((double)convertedFilesCount / (double)fileCount); return newFileName; } , new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = token } ); var parseCitizenInfo = new ActionBlock<string>( (fileName) => { var referendumProcessor = new ReferendumProcessor(); citizensOfArmenia.Post(referendumProcessor.ProcessFile(fileName)); } , new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = token } ); // Link the blocks citizensOfArmenia = new BufferBlock<List<Citizen>>(); getFileUrls.LinkTo(downloadFile, new DataflowLinkOptions { PropagateCompletion = true }); downloadFile.LinkTo(convertFile, new DataflowLinkOptions { PropagateCompletion = true }); convertFile.LinkTo(parseCitizenInfo, new DataflowLinkOptions { PropagateCompletion = true }); parseCitizenInfo.Completion.ContinueWith((tsk) => citizensOfArmenia.Complete()); }
public async static Task<Animat> SeedAsync(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry) { var yxzFileName = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) + ".txt"); bathymetry.ToYXZ(yxzFileName, -1); var mbs = new C3mbs(); mbsRESULT mbsResult; if (mbsRESULT.OK != (mbsResult = mbs.SetOutputDirectory(Path.GetTempPath()))) throw new AnimatInterfaceMMBSException("SetOutputDirectory Error:" + mbs.ResultToTc(mbsResult)); var config = mbs.GetConfiguration(); config.enabled = false; // binary output enabled/disabled config.durationLess = true; // make sure we're in durationless mode. mbs.SetConfiguration(config); mbsResult = mbs.LoadBathymetryFromTextFile(yxzFileName); if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("Bathymetry failed to load: " + mbs.ResultToTc(mbsResult)); mbsResult = mbs.AddSpecies(species.SpeciesDefinitionFilePath); if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException(string.Format("C3mbs::AddSpecies FATAL error {0} for species {1}", mbs.ResultToTc(mbsResult), species.SpeciesDefinitionFilePath)); var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest); var result = new Animat { ScenarioSpecies = species }; var area = bounds.Area; //Debug.WriteLine("Area: {0}",area); var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count => { var geos = new List<Geo<float>>(); for (var i = 0; i < count; i++) { var location = bounds.RandomLocationWithinPerimeter(); var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data; mbsRESULT retval; lock (mbs) retval = mbs.AddIndividualAnimat(0, new mbsPosition { latitude = location.Latitude, longitude = location.Longitude, depth = 0 }); if (mbsRESULT.OK == retval) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble()))); } return geos; }, new ExecutionDataflowBlockOptions { TaskScheduler = TaskScheduler.Default, BoundedCapacity = -1, MaxDegreeOfParallelism = -1, }); var bufferBlock = new BufferBlock<Geo<float>>(); transformManyBlock.LinkTo(bufferBlock); var population = (int)Math.Round(area * species.PopulationDensity); result.TotalAnimats = population; const int blockSize = 100; while (population > 0) { transformManyBlock.Post(population > blockSize ? blockSize : population); population -= blockSize; } transformManyBlock.Complete(); await transformManyBlock.Completion; //mbsResult = mbs.InitializeRun(); //if (mbsRESULT.OK == mbsResult) while (mbsRUNSTATE.INITIALIZING == mbs.GetRunState()) Thread.Sleep(1); //else throw new AnimatInterfaceMMBSException("C3mbs::Initialize FATAL error " + mbs.ResultToTc(mbsResult)); mbsResult = mbs.FinishRun(); if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("C3mbs::FinishRun FATAL error " + mbs.ResultToTc(mbsResult)); IList<Geo<float>> animatGeos; if (bufferBlock.TryReceiveAll(out animatGeos)) result.Locations.AddRange(animatGeos); return result; }
public static IPropagatorBlock<FileWord, FileWord> GetFileWordSaverBlock(Func<IRepository> repositoryFactory) { var inputBuffer = new BufferBlock<FileWord>(); var outputBuffer = new BufferBlock<FileWord>(); var batchBlockWithoutParents = new BatchBlock<FileWord>(1000); inputBuffer.LinkTo(batchBlockWithoutParents, i => i.File.FileId > 0 && i.Word.WordId > 0); inputBuffer.PropagateCompleted(batchBlockWithoutParents); var saveWithoutParentsBlock = new TransformManyBlock<FileWord[], FileWord>( async fileWords => { return await SaveEntities(repositoryFactory, fileWords, true); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism }); batchBlockWithoutParents.LinkToAndPropagateCompleted(saveWithoutParentsBlock); saveWithoutParentsBlock.LinkTo(outputBuffer); var batchBlockWithParents = new BatchBlock<FileWord>(300); // MaxMessages value means inputBuffer will unlink automatically from batchBlockWithParents after 300 messages. // This is required to prohibit "stealing" of workload from saveWithoutParentsBlock inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 }); inputBuffer.PropagateCompleted(batchBlockWithParents); var saveWithParentsBlock = new TransformManyBlock<FileWord[], FileWord>( async fileWords => { var res = await SaveEntities(repositoryFactory, fileWords, false); FileWord fileWord; if (inputBuffer.TryReceive(out fileWord)) // This unblocks inputBuffer due to unlinking from batchBlockWithParents { if (fileWord.File.FileId > 0 && fileWord.Word.WordId > 0) { batchBlockWithoutParents.Post(fileWord); } else { batchBlockWithParents.Post(fileWord); } } // Link again for another 300 messages inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 }); return res; }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 // If we are saving files/words also we cannot do this concurrently or duplicates of files/words may be inserted }); batchBlockWithParents.LinkToAndPropagateCompleted(saveWithParentsBlock); saveWithParentsBlock.LinkTo(outputBuffer); saveWithoutParentsBlock.Completion.ContinueWith( t => ((IDataflowBlock)outputBuffer).Fault(t.Exception), TaskContinuationOptions.OnlyOnFaulted); saveWithParentsBlock.Completion.ContinueWith( t => ((IDataflowBlock)outputBuffer).Fault(t.Exception), TaskContinuationOptions.OnlyOnFaulted); Task.WhenAll(saveWithoutParentsBlock.Completion, saveWithParentsBlock.Completion) .ContinueWith(t => outputBuffer.Complete(), TaskContinuationOptions.NotOnFaulted); return DataflowBlock.Encapsulate(inputBuffer, outputBuffer); }
public void RunTransformManyBlockConformanceTests() { bool passed = true; #region Sync { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory = options => { TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { return Enumerable.Range(0, 10); }); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return new[] { i }; }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test using a precanceled token { bool localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token }; var t = new TransformManyBlock<int, int>(i => new[] { i }, dbo); int ignoredValue; IList<int> ignoredValues; localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null; localPassed &= t.SendAsync(42).Result == false; localPassed &= t.TryReceiveAll(out ignoredValues) == false; localPassed &= t.Post(42) == false; localPassed &= t.OutputCount == 0; localPassed &= t.TryReceive(out ignoredValue) == false; localPassed &= t.Completion != null; t.Complete(); } catch (Exception) { localPassed = false; } Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, IEnumerable<int>>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList<int> list in new IList<int>[] { new int[1], new List<int>() { 0 }, new Collection<int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock<int, int> tmb1 = null; tmb1 = new TransformManyBlock<int, int>(i => { if (i == 1000) { tmb1.Complete(); return (IEnumerable<int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable<int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List<int>) { return new List<int>() { i + 1 }; } else return new Collection<int>() { i + 1 }; }, dbo); TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return i; }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock<int, int>(i => { if (i < ITERS - 1) throw new OperationCanceledException(); if (mode == 0) return new int[] { i }; else if (mode == 1) return new List<int>() { i }; else return Enumerable.Repeat(i, 1); }, dbo); var ab = new ActionBlock<int>(i => { if (i != ITERS - 1) localPassed = false; mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait(); mres.Wait(); } } } Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion #region Async { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory = options => { TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }), (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)Enumerable.Range(0, 10).ToArray())); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return new[] { i }; }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) return null; return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }); }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored when a reordering buffer is in place { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { if (i == 0) { Task.Delay(1000).Wait(); return null; } return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }); }), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); t.Post(0); t.Post(1); try { localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1; } catch { localPassed = false; } Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the delegate { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the task { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => Task<IEnumerable<int>>.Factory.StartNew(() => { throw new InvalidOperationException(); }))); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList<int> list in new IList<int>[] { new int[1], new List<int>() { 0 }, new Collection<int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock<int, int> tmb1 = null; tmb1 = new TransformManyBlock<int, int>(i => { return Task.Factory.StartNew(() => { if (i == 1000) { tmb1.Complete(); return (IEnumerable<int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable<int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List<int>) { return new List<int>() { i + 1 }; } else return new Collection<int>() { i + 1 }; }); }, dbo); TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return i; }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock<int, int>(i => { var cts = new CancellationTokenSource(); return Task.Factory.StartNew(() => { if (i < ITERS - 1) { cts.Cancel(); cts.Token.ThrowIfCancellationRequested(); } if (mode == 0) return new int[] { i }; else if (mode == 1) return new List<int>() { i }; else return Enumerable.Repeat(i, 1); }, cts.Token); }, dbo); var ab = new ActionBlock<int>(i => { if (i != ITERS - 1) localPassed = false; mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait(); mres.Wait(); } } } Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion Assert.True(passed, "Test failed."); }
public static async Task PurgeExpiredAsync(CloudBlobContainer inboxContainer) { Requires.NotNull(inboxContainer, "inboxContainer"); var deleteBlobsExpiringBefore = DateTime.UtcNow; int purgedBlobCount = 0; var searchExpiredBlobs = new TransformManyBlock<CloudBlobContainer, ICloudBlob>( async c => { try { var results = await c.ListBlobsSegmentedAsync( string.Empty, useFlatBlobListing: true, pageSize: 50, details: BlobListingDetails.Metadata, options: new BlobRequestOptions(), operationContext: null); return from blob in results.OfType<ICloudBlob>() let expires = DateTime.Parse(blob.Metadata[ExpirationDateMetadataKey]) where expires < deleteBlobsExpiringBefore select blob; } catch (StorageException ex) { var webException = ex.InnerException as WebException; if (webException != null) { var httpResponse = (HttpWebResponse)webException.Response; if (httpResponse.StatusCode == HttpStatusCode.NotFound) { // it's legit that some tests never created the container to begin with. return Enumerable.Empty<ICloudBlob>(); } } throw; } }, new ExecutionDataflowBlockOptions { BoundedCapacity = 4, }); var deleteBlobBlock = new ActionBlock<ICloudBlob>( blob => { Interlocked.Increment(ref purgedBlobCount); return blob.DeleteAsync(); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4, BoundedCapacity = 100, }); searchExpiredBlobs.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true }); searchExpiredBlobs.Post(inboxContainer); searchExpiredBlobs.Complete(); await deleteBlobBlock.Completion; }
public async Task TransformManyEnumerableToAction() { var data = new[] { 1 }; var tm = new TransformManyBlock<int, int>(i => data); int completedCount = 0; var c = new ActionBlock<int>(i => completedCount++); tm.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }); tm.PostRange(0, Iterations); tm.Complete(); await c.Completion; Assert.Equal(expected: Iterations, actual: completedCount); }
internal static void Run() { // // Create the members of the pipeline. // // Downloads the requested resource as a string. var downloadString = new TransformBlock <string, string>(async uri => { Console.WriteLine("Downloading '{0}'...", uri); return(await new HttpClient().GetStringAsync(uri)); }); // Separates the specified text into an array of words. var createWordList = new TransformBlock <string, string[]>(text => { Console.WriteLine("Creating word list..."); // Remove common punctuation by replacing all non-letter characters // with a space character. char[] tokens = text.Select(c => char.IsLetter(c) ? c : ' ').ToArray(); text = new string(tokens); // Separate the text into an array of words. return(text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)); }); // Removes short words and duplicates. var filterWordList = new TransformBlock <string[], string[]>(words => { Console.WriteLine("Filtering word list..."); return(words .Where(word => word.Length > 3) .Distinct() .ToArray()); }); // Finds all words in the specified collection whose reverse also // exists in the collection. var findReversedWords = new TransformManyBlock <string[], string>(words => { Console.WriteLine("Finding reversed words..."); // Dangerous to use with PLINQ var wordsSet = new HashSet <string>(words); return(from word in words.AsParallel() let reverse = new string(word.Reverse().ToArray()) where word != reverse && wordsSet.Contains(reverse) && wordsSet.Remove(word) && wordsSet.Remove(reverse) orderby word select word); }); // Prints the provided reversed words to the console. var printReversedWords = new ActionBlock <string>(reversedWord => { Console.WriteLine("Found reversed words {0}/{1}", reversedWord, new string(reversedWord.Reverse().ToArray())); }); // // Connect the dataflow blocks to form a pipeline. // var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; downloadString.LinkTo(createWordList, linkOptions); createWordList.LinkTo(filterWordList, linkOptions); filterWordList.LinkTo(findReversedWords, linkOptions); findReversedWords.LinkTo(printReversedWords, linkOptions); // Process "The Iliad of Homer" by Homer. downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt"); // Process The Bible downloadString.Post("https://www.gutenberg.org/files/30/30.txt"); downloadString.Complete(); // Wait for the last block in the pipeline to process all messages. printReversedWords.Completion.Wait(); }
public SimpleLogReader(DataflowOptions dataflowOptions) : base(dataflowOptions) { this.m_parsingBlock = new TransformManyBlock<string, string>( s => { if (string.IsNullOrEmpty(s)) { return Enumerable.Empty<string>(); } if (s == "ERROR") { throw new InvalidDataException("evil data :)"); } return s.Split(Splitor); }); this.m_recorder = new StatisticsRecorder(this); this.m_recordBlock = new ActionBlock<string>( s => { if (s == "ERROR") { throw new InvalidDataException("evil data :)"); } this.m_recorder.RecordEvent(s); }); var df1 = DataflowUtils.FromBlock(m_parsingBlock); var df2 = DataflowUtils.FromBlock(m_recordBlock); df1.LinkTo(df2); RegisterChild(df1); RegisterChild(df2); }
static void Main(string[] args) { // Downloads the requested resource as a string. var downloadString = new TransformBlock <string, string>(async uri => { Console.WriteLine("Downloading {0}...{1} #{2}", uri, DateTime.Now.ToString("HH:mm:ss fff"), Thread.CurrentThread.ManagedThreadId); return(await new HttpClient().GetStringAsync(uri)); }); // Separates the specified text into an array of words. var createWordlist = new TransformBlock <string, string[]>(text => { Console.WriteLine("Create word list...{0} #{1}", DateTime.Now.ToString("HH:mm:ss fff"), Thread.CurrentThread.ManagedThreadId); // Remove common punctuation by replacing all non-letter characters // with a space character. char[] token = text.Select(c => char.IsLetter(c) ? c : ' ').ToArray(); text = new string(token); // Separate the text into an array of words. return(text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)); }); // Removes short words and duplicates. var filterWordlist = new TransformBlock <string[], string[]>(words => { Console.WriteLine("Filter word list...{0} #{1}", DateTime.Now.ToString("HH:mm:ss fff"), Thread.CurrentThread.ManagedThreadId); return(words.Where(word => word.Length > 3) .Distinct().ToArray()); }); // Finds all words in the specified collection whose reverse also // exists in the collection. var findReverseWordlist = new TransformManyBlock <string[], string>(words => { Console.WriteLine("Find reversed word list...{0} {1}", DateTime.Now.ToString("HH:mm:ss fff"), Thread.CurrentThread.ManagedThreadId); var wordset = new HashSet <string>(words); return(from word in words.AsParallel() let reversed = new string(word.Reverse().ToArray()) where word != reversed && wordset.Contains(reversed) select word); }); // Prints the provided reversed words to the console. var printReversedWords = new ActionBlock <string>(reversed => { Console.WriteLine("Found reversed word: {0}/{1} #{2} {3}", reversed, new string(reversed.Reverse().ToArray()), Thread.CurrentThread.ManagedThreadId, DateTime.Now.ToString("HH:mm:ss fff")); }); // // Connect the dataflow blocks to form a pipeline. // var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; downloadString.LinkTo(createWordlist, linkOptions); createWordlist.LinkTo(filterWordlist, linkOptions); filterWordlist.LinkTo(findReverseWordlist, linkOptions); findReverseWordlist.LinkTo(printReversedWords, linkOptions); // Process "The Iliad of Homer" by Homer. downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt"); // Mark the head of the pipeline as complete. downloadString.Complete(); // Wait for the last block in the pipeline to process all messages. printReversedWords.Completion.Wait(); Console.WriteLine("Total thread {0}.", System.Diagnostics.Process.GetCurrentProcess().Threads.Count); Console.WriteLine("Press any key to exit."); Console.ReadKey(); }
static void Main(string[] args) { // // Create the members of the pipeline. // // Downloads the requested resource as a string. var downloadString = new TransformBlock<string, string>(uri => { Console.WriteLine("Downloading '{0}'...", uri); return new WebClient().DownloadString(uri); }); // Separates the specified text into an array of words. var createWordList = new TransformBlock<string, string[]>(text => { Console.WriteLine("Creating word list..."); // Remove common punctuation by replacing all non-letter characters // with a space character to. char[] tokens = text.ToArray(); for (int i = 0; i < tokens.Length; i++) { if (!char.IsLetter(tokens[i])) tokens[i] = ' '; } text = new string(tokens); // Separate the text into an array of words. return text.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); }); // Removes short words, orders the resulting words alphabetically, // and then remove duplicates. var filterWordList = new TransformBlock<string[], string[]>(words => { Console.WriteLine("Filtering word list..."); return words.Where(word => word.Length > 3).OrderBy(word => word) .Distinct().ToArray(); }); // Finds all words in the specified collection whose reverse also // exists in the collection. var findReversedWords = new TransformManyBlock<string[], string>(words => { Console.WriteLine("Finding reversed words..."); // Holds reversed words. var reversedWords = new ConcurrentQueue<string>(); // Add each word in the original collection to the result whose // reversed word also exists in the collection. Parallel.ForEach(words, word => { // Reverse the work. string reverse = new string(word.Reverse().ToArray()); // Enqueue the word if the reversed version also exists // in the collection. if (Array.BinarySearch<string>(words, reverse) >= 0 && word != reverse) { reversedWords.Enqueue(word); } }); return reversedWords; }); // Prints the provided reversed words to the console. var printReversedWords = new ActionBlock<string>(reversedWord => { Console.WriteLine("Found reversed words {0}/{1}", reversedWord, new string(reversedWord.Reverse().ToArray())); }); // // Connect the dataflow blocks to form a pipeline. // downloadString.LinkTo(createWordList); createWordList.LinkTo(filterWordList); filterWordList.LinkTo(findReversedWords); findReversedWords.LinkTo(printReversedWords); // // For each completion task in the pipeline, create a continuation task // that marks the next block in the pipeline as completed. // A completed dataflow block processes any buffered elements, but does // not accept new elements. // downloadString.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)createWordList).Fault(t.Exception); else createWordList.Complete(); }); createWordList.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)filterWordList).Fault(t.Exception); else filterWordList.Complete(); }); filterWordList.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)findReversedWords).Fault(t.Exception); else findReversedWords.Complete(); }); findReversedWords.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)printReversedWords).Fault(t.Exception); else printReversedWords.Complete(); }); // Process "The Iliad of Homer" by Homer. downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt"); // Mark the head of the pipeline as complete. The continuation tasks // propagate completion through the pipeline as each part of the // pipeline finishes. downloadString.Complete(); // Wait for the last block in the pipeline to process all messages. printReversedWords.Completion.Wait(); Console.ReadLine(); }
[InlineData(2, 1, false)] // no force ordered, but dop == 1, so it doesn't matter public async Task TestOrdering_Async_OrderedEnabled(int mmpt, int dop, bool? EnsureOrdered) { const int iters = 1000; var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, MaxMessagesPerTask = mmpt }; if (EnsureOrdered == null) { Assert.True(options.EnsureOrdered); } else { options.EnsureOrdered = EnsureOrdered.Value; } var tb = new TransformManyBlock<int, int>(i => Task.FromResult(Enumerable.Repeat(i, 1)), options); tb.PostRange(0, iters); for (int i = 0; i < iters; i++) { Assert.Equal(expected: i, actual: await tb.ReceiveAsync()); } tb.Complete(); await tb.Completion; }
public async Task TestOrdering_Sync_OrderedDisabled(bool trustedEnumeration) { // If ordering were enabled, this test would hang. var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, EnsureOrdered = false }; var mres = new ManualResetEventSlim(); var tb = new TransformManyBlock<int, int>(i => { if (i == 0) mres.Wait(); return trustedEnumeration ? new[] { i } : Enumerable.Repeat(i, 1); }, options); tb.Post(0); tb.Post(1); Assert.Equal(1, await tb.ReceiveAsync()); mres.Set(); Assert.Equal(0, await tb.ReceiveAsync()); tb.Complete(); await tb.Completion; }
static void Main(string[] args) { StringComparison comparison = StringComparison.InvariantCultureIgnoreCase; Predicate <string> linkFilter = link => link.IndexOf(".aspx", comparison) != -1 || link.IndexOf(".php", comparison) != -1 || link.IndexOf(".htm", comparison) != -1 || link.IndexOf(".html", comparison) != -1 || link.EndsWith(".com", comparison) || link.EndsWith(".net", comparison); Predicate <string> imgFilter = url => url.EndsWith(".jpg", comparison) || url.EndsWith(".png", comparison) || url.EndsWith(".gif", comparison); var downloader = new TransformBlock <string, string>( async(url) => { // using IOCP the thread pool worker thread does return to the pool var client = new HttpClient(); string result = await client.GetStringAsync(url); return(result); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 2 }); var contentBroadcaster = new BroadcastBlock <string>(s => s); var linkParser = new TransformManyBlock <string, string>( (html) => { var output = new List <string>(); var links = _linkRegexHRef.Matches(html); foreach (Match item in links) { var value = item.Groups["LINK"].Value; output.Add(value); } return(output); }); var imgParser = new TransformManyBlock <string, string>( (html) => { var output = new List <string>(); var images = _imgRegex.Matches(html); foreach (Match item in images) { var value = item.Groups["IMG"].Value; output.Add(value); } return(output); }); var writer = new ActionBlock <string>(async url => { var client = new HttpClient(); // using IOCP the thread pool worker thread does return to the pool byte[] buffer = await client.GetByteArrayAsync(url); string fileName = Path.GetFileName(url); Directory.CreateDirectory(AppDomain.CurrentDomain.BaseDirectory + @"\Images\"); string name = AppDomain.CurrentDomain.BaseDirectory + @"\Images\" + fileName; using (Stream srm = File.OpenWrite(name)) { await srm.WriteAsync(buffer, 0, buffer.Length); } }); var linkBroadcaster = new BroadcastBlock <string>(s => s); IDisposable disposeAll = new CompositeDisposable( // from [downloader] to [contentBroadcaster] downloader.LinkTo(contentBroadcaster), // from [contentBroadcaster] to [imgParser] contentBroadcaster.LinkTo(imgParser), // from [contentBroadcaster] to [linkParserHRef] contentBroadcaster.LinkTo(linkParser), // from [linkParser] to [linkBroadcaster] linkParser.LinkTo(linkBroadcaster), // conditional link to from [linkBroadcaster] to [downloader] linkBroadcaster.LinkTo(downloader, linkFilter), // from [linkBroadcaster] to [writer] linkBroadcaster.LinkTo(writer, imgFilter), // from [imgParser] to [writer] imgParser.LinkTo(writer)); downloader.Post("http://www.reactivex.io"); Console.WriteLine("Press any key to exit..."); Console.ReadLine(); downloader.Complete(); }
private static void DownloadBuilds(Args arguments) { s_logger.Info("Downloading Builds..."); // a couple of checks here Contract.Requires(arguments.NumBuilds > 0, "You must specify a number of builds"); Contract.Requires(arguments.Year > 0, "You must specify a year"); Contract.Requires(arguments.Month > 0, "You must specify a month"); Contract.Requires(arguments.Day > 0, "You must specify a day"); Contract.Requires(arguments.AppConfig != null, "You must specify a configuration file"); Contract.Requires(arguments.OutputDirectory != null, "You must specify an output directory"); Contract.Requires(File.Exists(arguments.AppConfig.AnalyzerConfig.Exe), "The analyzer executable file must exist"); Contract.Requires(Directory.Exists(arguments.OutputDirectory), "The output directory must exist"); // so in here we will create a new network. var buildInfoBlock = new TransformManyBlock <GetKustoBuildInput, List <KustoBuild> >(i => { var action = new GetKustoBuild(arguments.AppConfig); var result = action.PerformAction(i); // its not worth it to continue if this fails if (!result.ExecutionStatus) { s_logger.Error(result.Exception, "Could not download builds"); throw result.Exception; } return(result.Result.KustoBuildData); }); var downloadBlock = new TransformBlock <List <KustoBuild>, TimedActionResult <DecompressionOutput> >(i => { var action = new BuildDownload(arguments.AppConfig, arguments.OutputDirectory); return(action.PerformAction(i)); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxDownloadTasks } ); var analysisBlock = new ActionBlock <TimedActionResult <DecompressionOutput> >(i => { // check if (i.ExecutionStatus) { // analyze if decompression succeded var action = new BuildAnalisys(arguments.AppConfig); action.PerformAction(i.Result); } }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxAnalysisTasks } ); // link them buildInfoBlock.LinkTo(downloadBlock, new DataflowLinkOptions { PropagateCompletion = true }); downloadBlock.LinkTo(analysisBlock, new DataflowLinkOptions { PropagateCompletion = true }); // in here, we need to set up the days for which the downloads will be taken var date = new DateTime(arguments.Year, arguments.Month, arguments.Day); for (var i = 0; i < arguments.Span; ++i) { s_logger.Info($"Downloading builds from year={date.Year}, month={date.Month}, day={date.Day}"); // create the inout for this task var input = new GetKustoBuildInput(arguments.NumBuilds, date.Year, date.Month, date.Day); // post the task... buildInfoBlock.Post(input); // and add one more day date = date.AddDays(1); } // and complete buildInfoBlock.Complete(); // wait for the last... analysisBlock.Completion.Wait(); }
public async Task Process(CancellationToken cancellationToken) { DataflowLinkOptions linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; DataflowLinkOptions nonPropagatingLinkOptions = new DataflowLinkOptions(); ExecutionDataflowBlockOptions singleItemBlockOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 1, EnsureOrdered = false }; ExecutionDataflowBlockOptions bufferBlockOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 100, EnsureOrdered = false }; ExecutionDataflowBlockOptions unboundedBlockOptions = new ExecutionDataflowBlockOptions { EnsureOrdered = false }; GroupingDataflowBlockOptions batchingBlockOptions = new GroupingDataflowBlockOptions { EnsureOrdered = false }; TransformManyBlock <object, MxHostTestDetails> queuePoller = new TransformManyBlock <object, MxHostTestDetails>(_ => GetMxHostToProcess(_), singleItemBlockOptions); TransformBlock <MxHostTestDetails, MxHostTestDetails> retestPeriodFilter = new TransformBlock <MxHostTestDetails, MxHostTestDetails>((Func <MxHostTestDetails, MxHostTestDetails>)MarkTestToSkip, singleItemBlockOptions); BufferBlock <MxHostTestDetails> buffer = new BufferBlock <MxHostTestDetails>(bufferBlockOptions); TransformManyBlock <MxHostTestDetails, MxHostTestDetails> duplicateFilter = new TransformManyBlock <MxHostTestDetails, MxHostTestDetails>(_ => FilterHosts(_), singleItemBlockOptions); List <TransformBlock <MxHostTestDetails, MxHostTestDetails> > mxTestProcessors = Enumerable .Range(1, _config.TlsTesterThreadCount) .Select(index => new TransformBlock <MxHostTestDetails, MxHostTestDetails>(CreateTlsTester(Guid.NewGuid()), singleItemBlockOptions)) .ToList(); BatchBlock <MxHostTestDetails> resultBatcher = new BatchBlock <MxHostTestDetails>(_config.PublishBatchSize, batchingBlockOptions); Timer timer = new Timer(_ => { resultBatcher.TriggerBatch(); _log.LogDebug("Batch triggered."); }); TransformBlock <MxHostTestDetails, MxHostTestDetails> batchFlusher = new TransformBlock <MxHostTestDetails, MxHostTestDetails>(ResetTimer(timer), unboundedBlockOptions); TransformBlock <MxHostTestDetails[], MxHostTestDetails[]> resultPublisher = new TransformBlock <MxHostTestDetails[], MxHostTestDetails[]>(_ => PublishResults(_), unboundedBlockOptions); ActionBlock <MxHostTestDetails[]> deleteFromQueue = new ActionBlock <MxHostTestDetails[]>(_ => RemoveFromQueue(_), unboundedBlockOptions); queuePoller.LinkTo(retestPeriodFilter, linkOptions); retestPeriodFilter.LinkTo(batchFlusher, nonPropagatingLinkOptions, x => x.SkipTesting); retestPeriodFilter.LinkTo(buffer, linkOptions, x => !x.SkipTesting); buffer.LinkTo(duplicateFilter, linkOptions); mxTestProcessors.ForEach(processor => { duplicateFilter.LinkTo(processor, linkOptions); processor.LinkTo(batchFlusher, nonPropagatingLinkOptions); }); batchFlusher.LinkTo(resultBatcher, linkOptions); resultBatcher.LinkTo(resultPublisher, linkOptions); resultPublisher.LinkTo(deleteFromQueue, linkOptions); var blocks = new Dictionary <string, Func <int> > { ["Queued"] = () => queuePoller.OutputCount + buffer.Count, ["Processing"] = () => _processingFilter.HostCount, }; var processorTasks = mxTestProcessors.Select(processor => processor.Completion).ToArray(); // Start the stats print loop var statsTask = PrintStats(blocks, cancellationToken); await RunPipeline(queuePoller, cancellationToken); _log.LogInformation("Shutting down TLS Tester"); queuePoller.Complete(); _log.LogInformation("Waiting for test processors to complete..."); await Task.WhenAll(processorTasks); _log.LogInformation("Test processors complete. Flushing results..."); batchFlusher.Complete(); _log.LogInformation("Waiting for results flush and final shutdown..."); await Task.WhenAll( statsTask, deleteFromQueue.Completion ); _log.LogInformation("TLS tester shut down. Exiting."); }
private static async Task <int> Main() { string?knownArtistListFilename = Environment.GetCommandLineArgs().Skip(1).FirstOrDefault(); if (knownArtistListFilename == null) { Console.WriteLine("Please pass the filename of a newline-delimited text file containing a list of known artists to look up as the first argument."); return(1); } ISet <string> knownArtistNames = new HashSet <string>((await File.ReadAllLinesAsync(knownArtistListFilename, Encoding.UTF8)).Select(s => s.ToLowerInvariant())); var blockOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MAX_PARALLEL_DOWNLOADS }; TransformBlock <string, ArtistPage> fetchSource = new TransformBlock <string, ArtistPage>(async artistName => { Uri artistUri = FluentUriBuilder.From(BASE_URI.AbsoluteUri).Path(artistName.ToLowerInvariant()).ToUri(); using HttpResponseMessage response = await CLIENT.GetAsync(artistUri); var artistPage = new ArtistPage { artistName = artistName }; if (response.IsSuccessStatusCode) { artistPage.sourceCode = await response.Content.ReadAsStringAsync(); } else { Console.WriteLine($"Error: Failed to download artist page for {artistName}: {response.StatusCode}. Please check the artist name on {BASE_URI}"); } return(artistPage); }, blockOptions); TransformManyBlock <ArtistPage, Relationship> extractRelationships = new TransformManyBlock <ArtistPage, Relationship>(page => { if (page.sourceCode == null) { return(null); } MatchCollection artistNameMatches = Regex.Matches(page.sourceCode, @"<a href=""(?<artistSlug>.+?)"" class=S id=s\d+>(?<artistName>.+?)</a>"); IEnumerable <string> relatedArtistNames = artistNameMatches.Skip(1).Select(match => match.Groups["artistName"].Value); Match relationshipStrengthMatches = Regex.Match(page.sourceCode, @"Aid\[0\]=new Array\(-1(?:,(?<strength>-?\d+(?:\.\d+)?))*\);"); IEnumerable <double> relatedArtistStrengths = relationshipStrengthMatches.Groups["strength"].Captures.Select(capture => double.Parse(capture.Value)); return(relatedArtistNames.Zip(relatedArtistStrengths) .Select(tuple => new Relationship { knownArtist = page.artistName, unknownArtist = tuple.First, strength = tuple.Second })); }, blockOptions); IDictionary <string, double> strengthsByArtistName = new Dictionary <string, double>(); ActionBlock <Relationship> incoming = new ActionBlock <Relationship>(relationship => { if (knownArtistNames.Contains(relationship.unknownArtist.ToLowerInvariant())) { return; } strengthsByArtistName.TryGetValue(relationship.unknownArtist, out double existingStrength); strengthsByArtistName[relationship.unknownArtist] = existingStrength + relationship.strength; }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }); var outgoing = new BufferBlock <IDictionary <string, double> >(); #pragma warning disable 4014 incoming.Completion.ContinueWith(task => { outgoing.Post(strengthsByArtistName); }); #pragma warning restore 4014 IPropagatorBlock <Relationship, IDictionary <string, double> > reduce = DataflowBlock.Encapsulate(incoming, outgoing); ActionBlock <IDictionary <string, double> > printReport = new ActionBlock <IDictionary <string, double> >(strengths => { IOrderedEnumerable <KeyValuePair <string, double> > sorted = strengths.OrderByDescending(pair => pair.Value); foreach ((string unknownArtistName, double strength) in sorted) { Console.WriteLine($"{strength,5:N1}\t{unknownArtistName}"); } }); DataflowLinkOptions linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; fetchSource.LinkTo(extractRelationships, linkOptions); extractRelationships.LinkTo(reduce, linkOptions); reduce.LinkTo(printReport, linkOptions); foreach (string knownArtistName in knownArtistNames) { fetchSource.Post(knownArtistName); } fetchSource.Complete(); await printReport.Completion; return(0); }
public void ListWords() { // Create the members of the pipeline. Console.WriteLine("start"); // Downloads the requested resource as a string. var downloadString = new TransformBlock <string, string>(uri => downLoadStringFunc(uri)); // Separates the specified text into an array of words. var createWordList = new TransformBlock <string, string[]>(text => createWordListFunc(text)); // Removes short words, orders the resulting words alphabetically, // and then remove duplicates. var filterWordList = new TransformBlock <string[], string[]>(words => filterWorldListFunc(words)); // Finds all words in the specified collection whose reverse also // exists in the collection. var findReversedWords = new TransformManyBlock <string[], string>(words => { Console.WriteLine("Finding reversed words..."); // Holds reversed words. var reversedWords = new ConcurrentQueue <string>(); // Add each word in the original collection to the result whose // reversed word also exists in the collection. Parallel.ForEach(words, word => { // Reverse the work. string reverse = new string(word.Reverse().ToArray()); // Enqueue the word if the reversed version also exists // in the collection. if (Array.BinarySearch <string>(words, reverse) >= 0 && word != reverse) { reversedWords.Enqueue(word); } }); return(reversedWords); }); // Prints the provided reversed words to the console. var printReversedWords = new ActionBlock <string>(reversedWord => printReversedWordsFunc(reversedWord)); downloadString.LinkTo(createWordList); createWordList.LinkTo(filterWordList); filterWordList.LinkTo(findReversedWords); findReversedWords.LinkTo(printReversedWords); downloadString.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)createWordList).Fault(t.Exception); } else { createWordList.Complete(); } }); createWordList.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)filterWordList).Fault(t.Exception); } else { filterWordList.Complete(); } }); filterWordList.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)findReversedWords).Fault(t.Exception); } else { findReversedWords.Complete(); } }); findReversedWords.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)printReversedWords).Fault(t.Exception); } else { printReversedWords.Complete(); } }); downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt"); downloadString.Complete(); printReversedWords.Completion.Wait(); Console.Read(); }
internal static bool TransformManyEnumerableToAction() { const int ITERS = 2; var data = new[] { 1 }; var tm = new TransformManyBlock<int, int>(i => data); int completedCount = 0; var c = new ActionBlock<int>(i => completedCount++); tm.LinkWithCompletion(c); for (int i = 0; i < ITERS; i++) tm.Post(i); tm.Complete(); c.Completion.Wait(); return completedCount == ITERS; }
private static void DownloadMonthlyQueueData(Args arguments) { s_logger.Info($"Downloading Queue data ({(arguments.IncludeMachineMap? "including machine map" : "no machine map will be included")})"); // couple of checks here Contract.Requires(arguments.Year > 0, "You must specify a year"); Contract.Requires(arguments.Month > 0, "You must specify a month"); Contract.Requires(arguments.OutputDirectory != null, "You must specify an output directory"); Contract.Requires(Directory.Exists(arguments.OutputDirectory), "The output directory must exist"); if (arguments.IncludeMachineMap) { Contract.Requires(arguments.AppConfig.ConcurrencyConfig.MaxMachineMapTasks > 0, "You must specify a positive number of MaxMachineMapTasks"); } // and now download the data to a file var downloadQueueDataBlock = new TransformManyBlock <DownloadMonthlyQueueDataInput, KustoQueueData>( i => { var action = new DownloadMonthlyQueueData(arguments.AppConfig); var result = action.PerformAction(i); // its not worth it to continue if this fails if (!result.ExecutionStatus) { s_logger.Error(result.Exception, "Could not download queue data"); // just throw the exception here throw result.Exception; } return(result.Result.Queues); } ); // this is the one that builds machine maps var createMachineMapBlock = new ActionBlock <KustoQueueData>( i => { var action = new BuildQueueToMachineMap(arguments.AppConfig); var result = action.PerformAction(new BuildQueueToMachineMapInput(arguments.Year, arguments.Month, arguments.OutputDirectory, i.QueueName)); // its not worth it to continue if this fails if (!result.ExecutionStatus) { s_logger.Error(result.Exception, $"Could not create machine map for queue {i.QueueName}"); } }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxMachineMapTasks } ); // link downloadQueueDataBlock.LinkTo(createMachineMapBlock, new DataflowLinkOptions { PropagateCompletion = true }); // create the input var input = new DownloadMonthlyQueueDataInput(arguments.Year, arguments.Month, arguments.OutputDirectory); // post downloadQueueDataBlock.Post(input); // complete downloadQueueDataBlock.Complete(); // and wait if (!arguments.IncludeMachineMap) { downloadQueueDataBlock.Completion.Wait(); } else { createMachineMapBlock.Completion.Wait(); } // done }
public async Task TestOrdering_Async_OrderedDisabled(bool trustedEnumeration) { // If ordering were enabled, this test would hang. var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded, EnsureOrdered = false }; var tasks = new TaskCompletionSource<IEnumerable<int>>[10]; for (int i = 0; i < tasks.Length; i++) { tasks[i] = new TaskCompletionSource<IEnumerable<int>>(); } var tb = new TransformManyBlock<int, int>(i => tasks[i].Task, options); tb.PostRange(0, tasks.Length); for (int i = tasks.Length - 1; i >= 0; i--) { tasks[i].SetResult(trustedEnumeration ? new[] { i } : Enumerable.Repeat(i, 1)); Assert.Equal(expected: i, actual: await tb.ReceiveAsync()); } tb.Complete(); await tb.Completion; }
private static void ClassifyInstances(Args arguments) { // and a couple of checks here Contract.Requires(arguments.OutputDirectory != null, "You must specify an output directory"); Contract.Requires(arguments.InputDirectory != null, "You must specify an input directory"); Contract.Requires(Directory.Exists(arguments.OutputDirectory), "The output directory must exist"); Contract.Requires(Directory.Exists(arguments.InputDirectory), "The input directory must exist"); var sharedCount = 0; var nonSharedCount = 0; try { // load the classifier first s_logger.Info("Loading classifier..."); // work on the content store s_logger.Info($"Initializing store at [{arguments.OutputDirectory}]"); var store = new RocksDbContentPlacementPredictionStore(arguments.OutputDirectory, true); var opContext = new OperationContext(new Context(new LogWrapper(s_logger))); // init it var initialized = store.StartupAsync(opContext); initialized.Wait(); // and check if (!initialized.Result) { s_logger.Error($"Could not initialize RocksDbContentPlacementPredictionStore at [{arguments.OutputDirectory}]"); } var classifier = new ContentPlacementClassifier(arguments.AppConfig.ClassifierConfiguration); // create the pipeline. The first step here is to parse the input files, and we can do this in parallel var buildArtifactParsingBlock = new TransformManyBlock <ParseBuildArtifactsInput, KeyValuePair <string, IReadOnlyList <ArtifactWithBuildMeta> > >(i => { var action = new ParseBuildArtifacts(); var result = action.PerformAction(i); if (result.ExecutionStatus) { return(result.Result.ArtifactsByHash.ToList()); } else { s_logger.Error(result.Exception, $"Error when parsing [{i.BuildArtifactsFile}]"); throw result.Exception; } }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxBuildParsingTasks } ); // then, when we have one, we linearize it var linearizeBlock = new TransformBlock <KeyValuePair <string, IReadOnlyList <ArtifactWithBuildMeta> >, TimedActionResult <LinearizeArtifactsOutput> >(i => { var action = new LinearizeArtifacts(); return(action.PerformAction(new LinearizeArtifactsInput(i.Key, i.Value))); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxArtifactLinearizationTasks } ); // and we classify them var classifyBlock = new ActionBlock <TimedActionResult <LinearizeArtifactsOutput> >(i => { // i have an ml instance here if (i.ExecutionStatus) { var cpInstance = new ContentPlacementInstance() { Artifact = i.Result.Linear.AsInstance(), // using the default utility method QueueName = i.Result.Linear.Queues.First() // the first here is enough, since its always one! }; var result = classifier.Classify(cpInstance); if (result.Succeeded) { var selectedMachines = result.Value; foreach (var path in i.Result.Linear.ReportedPaths) { store.StoreResult(opContext, path, selectedMachines); Interlocked.Add(ref sharedCount, 1); } } else { Interlocked.Add(ref nonSharedCount, 1); } } }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxArtifactClassificationTasks } ); // link them var numParsingTasks = 0; buildArtifactParsingBlock.LinkTo(linearizeBlock, new DataflowLinkOptions { PropagateCompletion = true }); linearizeBlock.LinkTo(classifyBlock, new DataflowLinkOptions { PropagateCompletion = true }); // do now we can post to the initial queue foreach (var file in Directory.EnumerateFiles(arguments.InputDirectory, "*.json")) { buildArtifactParsingBlock.Post(new ParseBuildArtifactsInput(file)); ++numParsingTasks; } s_logger.Info($"Posted {numParsingTasks} parsing tasks, processing"); // now wait buildArtifactParsingBlock.Complete(); classifyBlock.Completion.Wait(); // and now we should snapshot var snapshotDir = Path.Combine(arguments.OutputDirectory, "Snap"); Directory.CreateDirectory(snapshotDir); s_logger.Info($"Done, snapshoting to [{snapshotDir}]"); var result = store.CreateSnapshot(opContext, snapshotDir); // done } finally { var total = 1.0 * (sharedCount + nonSharedCount); var percentage = (1.0 * sharedCount) / total; s_logger.Info($"Stats: shared={sharedCount} ({percentage}), nonShared={nonSharedCount}, total={total}"); } }
private static async Task Main(string[] args) { var baseUri = new Uri(args[0]); var outputSize = args.Length > 1 ? int.Parse(args[1]) : DefaultOutputSize; var downloadString = new TransformBlock <Uri, string>( async uri => { using var client = new HttpClient(); return(await client.GetStringAsync(uri)); }); var visitedUris = new HashSet <Uri>(); var findUris = new TransformManyBlock <string, Uri>( html => AnchorRef.Matches(html) .Cast <Match>() .Select(m => new Uri(m.Value, UriKind.RelativeOrAbsolute)) .Select(uri => uri.IsAbsoluteUri ? uri : new Uri(baseUri, uri)) .Where(uri => visitedUris.Add(uri)), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, }); var broadcastUris = new BroadcastBlock <Uri>(uri => uri); var collectResults = new BatchBlock <Uri>(outputSize); downloadString.LinkTo(findUris); findUris.LinkTo(broadcastUris); broadcastUris.LinkTo(collectResults); broadcastUris.LinkTo( downloadString, uri => baseUri.IsBaseOf(uri) && uri != baseUri); downloadString.Post(baseUri); try { var uris = await collectResults.ReceiveAsync(TimeSpan.FromSeconds(5)); PrintResults(uris); } catch (InvalidOperationException exception) // time out { PrintException(exception); } finally { downloadString.Complete(); } try { await downloadString.Completion; } catch (AggregateException exception) { PrintException(exception); } catch (Exception exception) { PrintException(exception); } }
/// <inheritdoc/> public async Task RunAsync(ProcessMode processMode, CancellationToken cancellationToken) { if (_messageEncoder == null) { throw new NotInitializedException(); } try { if (IsRunning) { return; } IsRunning = true; _encodingBlock = new TransformManyBlock <DataSetMessageModel[], NetworkMessageModel>( async input => { try { if (_dataSetMessageBufferSize == 1) { return(await _messageEncoder.EncodeAsync(input, _maxEncodedMessageSize - _encodedMessageSizeOverhead).ConfigureAwait(false)); } else { return(await _messageEncoder.EncodeBatchAsync(input, _maxEncodedMessageSize - _encodedMessageSizeOverhead).ConfigureAwait(false)); } } catch (Exception e) { _logger.Error(e, "Encoding failure"); return(Enumerable.Empty <NetworkMessageModel>()); } }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, }); _batchDataSetMessageBlock = new BatchBlock <DataSetMessageModel>( _dataSetMessageBufferSize, new GroupingDataflowBlockOptions { CancellationToken = cancellationToken, }); _batchNetworkMessageBlock = new BatchBlock <NetworkMessageModel>( _networkMessageBufferSize, new GroupingDataflowBlockOptions { CancellationToken = cancellationToken, }); _sinkBlock = new ActionBlock <NetworkMessageModel[]>( async input => { if (input != null && input.Any()) { await _messageSink.SendAsync(input).ConfigureAwait(false); } else { _logger.Information("Sink block in engine {Name} triggered with empty input", Name); } }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, }); _batchDataSetMessageBlock.LinkTo(_encodingBlock); _encodingBlock.LinkTo(_batchNetworkMessageBlock); _batchNetworkMessageBlock.LinkTo(_sinkBlock); _messageTrigger.OnMessage += MessageTriggerMessageReceived; if (_diagnosticInterval > TimeSpan.Zero) { _diagnosticsOutputTimer.Change(_diagnosticInterval, _diagnosticInterval); } await _messageTrigger.RunAsync(cancellationToken).ConfigureAwait(false); } finally { IsRunning = false; _messageTrigger.OnMessage -= MessageTriggerMessageReceived; _diagnosticsOutputTimer.Change(Timeout.Infinite, Timeout.Infinite); _batchTriggerIntervalTimer.Change(Timeout.Infinite, Timeout.Infinite); await _sinkBlock.Completion; } }
private static ITargetBlock <string> CreateTextProcessingPipeline(string inputPath, out Task completionTask) { int fileCount = Directory.GetFiles(inputPath, "*.txt").Length; var getFilenames = new TransformManyBlock <string, string>( path => { return(Directory.GetFiles(path, "*.txt")); }); var getFileContents = new TransformBlock <string, string>(async(filename) => { Console.WriteLine("Begin: getFileContents"); using var streamReader = new StreamReader(filename); return(await streamReader.ReadToEndAsync()); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }); var analyzeContents = new TransformBlock <string, Dictionary <string, ulong> >(contents => { Console.WriteLine("Begin: analyzeContents;"); var frequencies = new Dictionary <string, ulong>(10000, StringComparer.OrdinalIgnoreCase); var matches = WordRegex.Matches(contents); foreach (Match match in matches) { if (!frequencies.TryGetValue(match.Value, out ulong currentValue)) { currentValue = 0; } frequencies[match.Value] = currentValue + 1; } Console.WriteLine("End: analyzeContents"); return(frequencies); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }); var eliminateIgnoredWords = new TransformBlock <Dictionary <string, ulong>, Dictionary <string, ulong> >(input => { foreach (var word in IgnoreWords) { input.Remove(word); } return(input); }); var batch = new BatchBlock <Dictionary <string, ulong> >(fileCount); // 所有的处理都集中在这一个点上 var combineFrequencies = new TransformBlock <Dictionary <string, ulong>[], List <KeyValuePair <string, ulong> > >(inputs => { Console.WriteLine("Begin: combiningFrequencies"); var sortedList = new List <KeyValuePair <string, ulong> >(); var combineFrequencies = new Dictionary <string, ulong>(10000, StringComparer.OrdinalIgnoreCase); foreach (var input in inputs) { foreach (var kvp in input) { if (!combineFrequencies.TryGetValue(kvp.Key, out ulong currentFrequency)) { currentFrequency = 0; } combineFrequencies[kvp.Key] = currentFrequency + kvp.Value; } } foreach (var kvp in combineFrequencies) { sortedList.Add(kvp); } sortedList.Sort((a, b) => { return(-a.Value.CompareTo(b.Value)); }); Console.WriteLine("End: combineFrequencies"); return(sortedList); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }); var printTopTen = new ActionBlock <List <KeyValuePair <string, ulong> > >(input => { for (int i = 0; i < input.Count; i++) { Console.WriteLine($"{input[i].Key} - {input[i].Value}"); } getFilenames.Complete(); }); // 链接块 getFilenames.LinkTo(getFileContents); getFileContents.LinkTo(analyzeContents); analyzeContents.LinkTo(eliminateIgnoredWords); eliminateIgnoredWords.LinkTo(batch); batch.LinkTo(combineFrequencies); combineFrequencies.LinkTo(printTopTen); completionTask = getFilenames.Completion; return(getFilenames); }
public void RunTransformManyBlockConformanceTests() { bool passed = true; #region Sync { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++) { Func <DataflowBlockOptions, TargetProperties <int> > transformManyBlockFactory = options => { TransformManyBlock <int, int> transformManyBlock = new TransformManyBlock <int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options); ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return(new TargetProperties <int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }); }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { localPassed &= network.Post(i) == true; } for (int i = 0; i < ITERS; i++) { localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16; } Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 })); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) { tasks[i - 1] = network.SendAsync(i); } Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) { total += ((IReceivableSourceBlock <int>)network).Receive(); } localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => { return(Enumerable.Range(0, 10)); }); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => { if ((i % 2) == 0) { throw new OperationCanceledException(); } return(new[] { i }); }); for (int i = 0; i < 10; i++) { t.Post(i); } t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) { localPassed &= t.Receive() == i; } } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test using a precanceled token { bool localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token }; var t = new TransformManyBlock <int, int>(i => new[] { i }, dbo); int ignoredValue; IList <int> ignoredValues; localPassed &= t.LinkTo(new ActionBlock <int>(delegate { })) != null; localPassed &= t.SendAsync(42).Result == false; localPassed &= t.TryReceiveAll(out ignoredValues) == false; localPassed &= t.Post(42) == false; localPassed &= t.OutputCount == 0; localPassed &= t.TryReceive(out ignoredValue) == false; localPassed &= t.Completion != null; t.Complete(); } catch (Exception) { localPassed = false; } Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting { bool localPassed = true; var t = new TransformManyBlock <int, int>(new Func <int, IEnumerable <int> >(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList <int> list in new IList <int>[] { new int[1], new List <int>() { 0 }, new Collection <int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock <int, int> tmb1 = null; tmb1 = new TransformManyBlock <int, int>(i => { if (i == 1000) { tmb1.Complete(); return((IEnumerable <int>)null); } else if (dop == 1) { list[0] = i + 1; return((IEnumerable <int>)list); } else if (list is int[]) { return(new int[1] { i + 1 }); } else if (list is List <int> ) { return(new List <int>() { i + 1 }); } else { return new Collection <int>() { i + 1 } }; }, dbo); TransformBlock <int, int> tmb2 = new TransformBlock <int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return(i); }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock <int, int>(i => { if (i < ITERS - 1) { throw new OperationCanceledException(); } if (mode == 0) { return new int[] { i } } ; else if (mode == 1) { return new List <int>() { i } } ; else { return(Enumerable.Repeat(i, 1)); } }, dbo); var ab = new ActionBlock <int>(i => { if (i != ITERS - 1) { localPassed = false; } mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) { tmb.SendAsync(i).Wait(); } mres.Wait(); } } } Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion #region Async { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++) { Func <DataflowBlockOptions, TargetProperties <int> > transformManyBlockFactory = options => { TransformManyBlock <int, int> transformManyBlock = new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i }), (ExecutionDataflowBlockOptions)options); ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return(new TargetProperties <int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }); }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { localPassed &= network.Post(i) == true; } for (int i = 0; i < ITERS; i++) { localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16; } Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 }))); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) { tasks[i - 1] = network.SendAsync(i); } Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) { total += ((IReceivableSourceBlock <int>)network).Receive(); } localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>)Enumerable.Range(0, 10).ToArray())); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => { if ((i % 2) == 0) { throw new OperationCanceledException(); } return(new[] { i }); }); for (int i = 0; i < 10; i++) { t.Post(i); } t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) { localPassed &= t.Receive() == i; } } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => { if ((i % 2) == 0) { return(null); } return(Task.Run(() => (IEnumerable <int>) new[] { i })); }); for (int i = 0; i < 10; i++) { t.Post(i); } t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) { localPassed &= t.Receive() == i; } } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored when a reordering buffer is in place { bool localPassed = true; var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => { if (i == 0) { Task.Delay(1000).Wait(); return(null); } return(Task.Run(() => (IEnumerable <int>) new[] { i })); }), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); t.Post(0); t.Post(1); try { localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1; } catch { localPassed = false; } Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the delegate { bool localPassed = true; var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the task { bool localPassed = true; var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => Task.Run <IEnumerable <int> >(new Func <IEnumerable <int> >(() => { throw new InvalidOperationException(); })))); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList <int> list in new IList <int>[] { new int[1], new List <int>() { 0 }, new Collection <int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock <int, int> tmb1 = null; tmb1 = new TransformManyBlock <int, int>(i => { return(Task.Run(() => { if (i == 1000) { tmb1.Complete(); return (IEnumerable <int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable <int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List <int> ) { return new List <int>() { i + 1 }; } else { return new Collection <int>() { i + 1 } }; })); }, dbo); TransformBlock <int, int> tmb2 = new TransformBlock <int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return(i); }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock <int, int>(i => { var cts = new CancellationTokenSource(); return(Task.Run(() => { if (i < ITERS - 1) { cts.Cancel(); cts.Token.ThrowIfCancellationRequested(); } if (mode == 0) { return new int[] { i } } ; else if (mode == 1) { return new List <int>() { i } } ; else { return Enumerable.Repeat(i, 1); } }, cts.Token)); }, dbo); var ab = new ActionBlock <int>(i => { if (i != ITERS - 1) { localPassed = false; } mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) { tmb.SendAsync(i).Wait(); } mres.Wait(); } } } Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion Assert.True(passed, "Test failed."); }
/// <summary> /// Creates a dataflow block from current configuration. /// </summary> protected override IPropagatorBlock <TInput, TOutput> CreateBlock() { TransformManyBlock <TInput, TOutput> block; if (this.processAsync != null) { block = new TransformManyBlock <TInput, TOutput> (async input => { // ReSharper disable once CompareNonConstrainedGenericWithNull if (input == null) { return(new TOutput[0]); } try { var result = await this.processAsync(input).ConfigureAwait(false); return(result); } catch (Exception ex) { var logger = input as IDataflowErrorLogger; if (logger != null) { logger.OnException(ex); } else { this.DefaultExceptionLogger?.Invoke(ex, input); } return(new TOutput[0]); } }, this.options); } else { block = new TransformManyBlock <TInput, TOutput> (input => { // ReSharper disable once CompareNonConstrainedGenericWithNull if (input == null) { return(new TOutput[0]); } try { var result = this.processSync(input); return(result); } catch (Exception ex) { var logger = input as IDataflowErrorLogger; if (logger != null) { logger.OnException(ex); } else { this.DefaultExceptionLogger?.Invoke(ex, input); } return(new TOutput[0]); } }, this.options); } return(block); }
public async Task HandleAsync(CommandContext context, NextDelegate next) { if (context.Command is BulkUpdateContents bulkUpdates) { if (bulkUpdates.Jobs?.Length > 0) { var executionOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2) }; var createCommandsBlock = new TransformManyBlock <BulkTask, BulkTaskCommand>(async task => { try { return(await CreateCommandsAsync(task)); } catch (OperationCanceledException ex) { // Dataflow swallows operation cancelled exception. throw new AggregateException(ex); } }, executionOptions); var executeCommandBlock = new ActionBlock <BulkTaskCommand>(async command => { try { await ExecuteCommandAsync(command); } catch (OperationCanceledException ex) { // Dataflow swallows operation cancelled exception. throw new AggregateException(ex); } }, executionOptions); createCommandsBlock.BidirectionalLinkTo(executeCommandBlock); contextProvider.Context.Change(b => b .WithoutContentEnrichment() .WithoutCleanup() .WithUnpublished(true) .WithoutTotal()); var requestedSchema = bulkUpdates.SchemaId.Name; var results = new ConcurrentBag <BulkUpdateResultItem>(); for (var i = 0; i < bulkUpdates.Jobs.Length; i++) { var task = new BulkTask( context.CommandBus, requestedSchema, i, bulkUpdates.Jobs[i], bulkUpdates, results); if (!await createCommandsBlock.SendAsync(task)) { break; } } createCommandsBlock.Complete(); await executeCommandBlock.Completion; context.Complete(new BulkUpdateResult(results)); } else { context.Complete(new BulkUpdateResult()); } } else { await next(context); } }
public void Dispose() { if (this.evalBlock != null) { this.evalBlock.Complete(); this.evalBlock.Completion.Wait(TimeSpan.FromMinutes(1)); this.evalBlock = null; } if (this.evalBlockDisposable != null) { this.evalBlockDisposable.Dispose(); this.evalBlockDisposable = null; } }
/// <summary> /// Purges all blobs set to expire prior to the specified date. /// </summary> /// <param name="deleteBlobsExpiringBefore"> /// All blobs scheduled to expire prior to this date will be purged. The default value /// is interpreted as <see cref="DateTime.UtcNow"/>. /// </param> /// <returns>The task representing the asynchronous operation.</returns> public async Task PurgeBlobsExpiringBeforeAsync(DateTime deleteBlobsExpiringBefore = default(DateTime)) { if (deleteBlobsExpiringBefore == default(DateTime)) { deleteBlobsExpiringBefore = DateTime.UtcNow; } Requires.Argument(deleteBlobsExpiringBefore.Kind == DateTimeKind.Utc, "expirationUtc", "UTC required."); var searchExpiredDirectoriesBlock = new TransformManyBlock<CloudBlobContainer, CloudBlobDirectory>( async c => { var results = await c.ListBlobsSegmentedAsync(); return from directory in results.OfType<CloudBlobDirectory>() let expires = DateTime.Parse(directory.Uri.Segments[directory.Uri.Segments.Length - 1].TrimEnd('/')) where expires < deleteBlobsExpiringBefore select directory; }, new ExecutionDataflowBlockOptions { BoundedCapacity = 4, }); var deleteDirectoryBlock = new TransformManyBlock<CloudBlobDirectory, CloudBlockBlob>( async directory => { var results = await directory.ListBlobsSegmentedAsync(); return results.OfType<CloudBlockBlob>(); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, BoundedCapacity = 4, }); var deleteBlobBlock = new ActionBlock<CloudBlockBlob>( blob => blob.DeleteAsync(), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4, BoundedCapacity = 100, }); searchExpiredDirectoriesBlock.LinkTo(deleteDirectoryBlock, new DataflowLinkOptions { PropagateCompletion = true }); deleteDirectoryBlock.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true }); searchExpiredDirectoriesBlock.Post(this.container); searchExpiredDirectoriesBlock.Complete(); await deleteBlobBlock.Completion; }
public void AsyncCancelledTest () { var scheduler = new TestScheduler (); var block = new TransformManyBlock<int, int> ( i => { var tcs = new TaskCompletionSource<IEnumerable<int>> (); tcs.SetCanceled (); return tcs.Task; }, new ExecutionDataflowBlockOptions { TaskScheduler = scheduler }); Assert.IsTrue (block.Post (1)); scheduler.ExecuteAll (); Assert.IsFalse (block.Completion.Wait (100)); }