/// <summary>
/// Search Asynchrony many extension in all of Fixed and Removable Disks.
/// </summary>
/// <param name="targetExtensions">Some file extensions for use search pattern.</param>
/// <example>
/// FileExtension example:
/// {".jpg", 646546Byte, 646Byte}
/// {".pdf", 25464645546Byte, 60000Byte}
/// </example>
/// <returns>A sorted list of detected files</returns>
public static async Task<List<FileInfo>> DiskParallelProbingAsync(List<FileExtensionOption> targetExtensions, System.Threading.CancellationTokenSource CTS)
{
return await Task.Run(() =>
{
searchComplete = false;
//
Reporter("DiskProbing", new ReportEventArgs("DiskProbing", ReportCodes.DiskProbingStarted, "---{Search Disks Started}---"));
List<FileInfo> _result = new List<FileInfo>();
//
// Find specific folders from windows drives instead of the total drive.
//
FolderInfo[] SpecificsDirectory = CheckDirectoriesChanges.GetDirectoriesInformation();
//
// Set Data-flow
//
TransformBlock<FolderInfo, List<FileInfo>> TB = new TransformBlock<FolderInfo, List<FileInfo>>(dir =>
{
Reporter(dir, new ReportEventArgs("DiskProbing",
ReportCodes.TheSearchBeginning,
"Searching {0} ...", dir.FullName));
List<FileInfo> res = dir.GetDirectoryInfo.SearchDirectory(targetExtensions, CTS);
Reporter(dir, new ReportEventArgs("DiskProbing",
ReportCodes.TheSearchCompleted,
"The Search {0} was completed!", dir.FullName));
return res;
}, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount });
ActionBlock<List<FileInfo>> AB = new ActionBlock<List<FileInfo>>(lst => _result.AddRange(lst));
//
// Search specific folders from windows drives instead of the total drive.
//
try
{
TB.LinkTo(AB);
ParallelOptions opt = new ParallelOptions() { CancellationToken = CTS.Token, MaxDegreeOfParallelism = Environment.ProcessorCount };
var pLoop = Parallel.ForEach(SpecificsDirectory, opt, async dir => await TB.SendAsync(dir));
TB.Complete();
TB.Completion.Wait();
}
catch (Exception ex) { Reporter(SpecificsDirectory, new ReportEventArgs("SearchEngine.DiskProbing.SpecificsDirectory", ex)); }
searchComplete = true;
Reporter("DiskProbing", new ReportEventArgs("DiskProbing",
ReportCodes.DiskProbingFinished,
"---{Search Disks Finished}---"));
LastSearchResult = _result;
return _result;
});
}
static public void ProcessingByTPL_StraightForwardImplementation()
{
const string pathToFiles = @"..\..\..\..\DataFiles";
string[] files = Directory.GetFiles(pathToFiles, "*.txt", SearchOption.AllDirectories);
var loadDataFromFileBlock = new TransformBlock<string[], List<CustomerTextData>>(fileItems =>
{
var factory = new CustomerTextDataFactory();
return new List<CustomerTextData>(Array.ConvertAll(fileItems, factory.LoadFromFile));
});
var filterBlock = new TransformBlock<List<CustomerTextData>, List<CustomerTextData>>(textDataList =>
{
var filter = new FilterTextData(5);
return textDataList.Where(filter.Run).ToList();
});
var toListBlock = new TransformManyBlock<List<CustomerTextData>, CustomerTextData>(textDataList =>
{
var queue = new ConcurrentQueue<CustomerTextData>();
textDataList.ForEach(queue.Enqueue);
return queue;
});
var action = new ActionBlock<CustomerTextData>(textData =>
{
var weight = new WeightTextData();
int result = weight.Run(textData);
Trace.WriteLine(result);
Console.WriteLine(result);
});
loadDataFromFileBlock.LinkTo(filterBlock);
filterBlock.LinkTo(toListBlock);
toListBlock.LinkTo(action);
loadDataFromFileBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterBlock).Fault(t.Exception);
else filterBlock.Complete();
});
filterBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)toListBlock).Fault(t.Exception);
else toListBlock.Complete();
});
toListBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)action).Fault(t.Exception);
else action.Complete();
});
loadDataFromFileBlock.Post(files);
loadDataFromFileBlock.Complete();
action.Completion.Wait();
}
static void Main(string[] args)
{
string s =
"http://cn.bing.com/search?q=MD5CryptoServiceProvider+slow&qs=n&pq=md5cryptoserviceprovider+slow&sc=0-25&sp=-1&sk=&cvid=67d40cbd8c424d55a3db83e6e9868267&first=51&FORM=PERE4";
using (MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider())
{
byte[] inBytes = Encoding.UTF8.GetBytes(s);
var bytes = md5.ComputeHash(inBytes);
Console.WriteLine(bytes.Length);
}
var splitter = new TransformBlock<string, KeyValuePair<string, int>>(
input =>
{
var splitted = input.Split('=');
return new KeyValuePair<string, int>(splitted[0], int.Parse(splitted[1]));
});
var dict = new Dictionary<string, int>();
var aggregater = new ActionBlock<KeyValuePair<string, int>>(
pair =>
{
int oldValue;
dict[pair.Key] = dict.TryGetValue(pair.Key, out oldValue) ? oldValue + pair.Value : pair.Value;
});
splitter.LinkTo(aggregater, new DataflowLinkOptions() { PropagateCompletion = true});
splitter.Post("a=1");
splitter.Post("b=2");
splitter.Post("a=5");
splitter.Complete();
aggregater.Completion.Wait();
Console.WriteLine("sum(a) = {0}", dict["a"]); //prints sum(a) = 6
//CalcAsync().Wait();
//SlowFlowAsync().Wait();
//FailDemoAsync().Wait();
//TransformAndLinkDemo().Wait();
//LinkLeftToDemo().Wait();
//CircularFlowAutoComplete().Wait();
//RecorderDemo().Wait();
BulkInserterDemo().Wait();
//BulkInserterDemo2().Wait();
//BroadcasterDemo().Wait();
//MyLoggerDemo().Wait();
//ETLLookupDemo().Wait();
}
public async Task TransformThroughFilterToAction()
{
int completedCount = 0;
var t = new TransformBlock<int, int>(i => i);
var c = new ActionBlock<int>(i => completedCount++);
t.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }, i => true);
t.PostRange(0, Iterations);
t.Complete();
await c.Completion;
Assert.Equal(expected: Iterations, actual: completedCount);
}
internal static bool TransformThroughFilterToAction()
{
const int ITERS = 2;
int completedCount = 0;
var t = new TransformBlock<int, int>(i => i);
var c = new ActionBlock<int>(i => completedCount++);
t.LinkTo(c, i => true);
t.Completion.ContinueWith(_ => c.Complete());
for (int i = 0; i < ITERS; i++) t.Post(i);
t.Complete();
c.Completion.Wait();
return completedCount == ITERS;
}
public void Run()
{
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 };
var tb = new TransformBlock<int, int>(i => i * 2);
var ab = new ActionBlock<int>(i => this.Compute(i), options);
tb.LinkTo(ab);
for (var i = 0; i < 10; i++)
{
tb.Post(i);
}
tb.Complete();
tb.Completion.Wait();
Thread.Sleep(500);
}
public async Task TransformToAction()
{
var t = new TransformBlock<int, int>(i => i * 2);
int completedCount = 0;
int prev = -2;
var c = new ActionBlock<int>(i =>
{
completedCount++;
Assert.Equal(expected: i, actual: prev + 2);
prev = i;
});
t.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true });
t.PostRange(0, Iterations);
t.Complete();
await c.Completion;
Assert.True(completedCount == Iterations);
}
public async Task TenTransformsToAction()
{
var first = new TransformBlock<int, int>(item => item);
TransformBlock<int, int> t = first;
for (int i = 0; i < 9; i++)
{
var next = new TransformBlock<int, int>(item => item);
t.LinkTo(next, new DataflowLinkOptions { PropagateCompletion = true });
t = next;
}
int completedCount = 0;
var last = new ActionBlock<int>(i => completedCount++);
t.LinkTo(last, new DataflowLinkOptions { PropagateCompletion = true });
first.PostRange(0, Iterations);
first.Complete();
await last.Completion;
Assert.Equal(expected: Iterations, actual: completedCount);
}
public async static void TData()
{
var multiplyBlock = new TransformBlock<int, int>(item =>
{
var res = item * 2;
Console.WriteLine("{0} * 2 = {1}", item, res);
return res;
});
var divideBlock = new TransformBlock<int, int>(item =>
{
var res = item / 2;
Console.WriteLine("{0} / 2 = {1}", item, res);
return res;
});
multiplyBlock.LinkTo(divideBlock);
multiplyBlock.Post(2);
multiplyBlock.Complete();
await divideBlock.Completion;
}
internal static bool TransformToAction()
{
bool passed = true;
const int ITERS = 2;
var t = new TransformBlock<int, int>(i => i * 2);
int completedCount = 0;
int prev = -2;
var c = new ActionBlock<int>(i =>
{
completedCount++;
if (i != prev + 2) passed &= false;
prev = i;
});
t.LinkWithCompletion(c);
for (int i = 0; i < ITERS; i++) t.Post(i);
t.Complete();
c.Completion.Wait();
Assert.True(completedCount == ITERS);
return passed;
}
static void Main(string[] args)
{
var multiplyBlock = new TransformBlock<int, int>(value => value * 2);
var subtractBlock = new TransformBlock<int, int>(value => value - 2);
var displayBlock = new ActionBlock<int>(value => Console.WriteLine(value));
// multiplyBlock ==> subtractBlock ==> displayBlock
var linkOptions = new DataflowLinkOptions { PropagateCompletion = true };
multiplyBlock.LinkTo(subtractBlock, linkOptions);
subtractBlock.LinkTo(displayBlock, linkOptions);
// Put data in the first block (multiplyBlock)
foreach (var i in Enumerable.Range(0, 10))
multiplyBlock.Post(i);
// Mark it as complete. Completion will propagate because of the link options.
multiplyBlock.Complete();
// Wait for the last block (displayBlock) to complete.
displayBlock.Completion.Wait();
Console.ReadKey();
}
public void Reset()
{
_session?.Dispose();
_builder?.Complete();
_builder = null;
}
private void WorkerTimerElapsed(CancellationToken cancellationToken)
{
if (!_storageInfoProvider.HasSpaceAvailableForExport)
{
_logger.Log(LogLevel.Warning, $"Export service paused due to insufficient storage space. Available storage space: {_storageInfoProvider.AvailableFreeSpace:D}.");
return;
}
var downloadActionBlock = new TransformBlock <string, IList <TaskResponse> >(
async(agent) => await DownloadActionCallback(agent, cancellationToken));
var dataConvertTransformBlock = new TransformManyBlock <IList <TaskResponse>, OutputJob>(
(tasks) =>
{
if (tasks.IsNullOrEmpty())
{
return(null);
}
return(ConvertDataBlockCallback(tasks, cancellationToken));
});
var downloadPayloadTransformBlock = new TransformBlock <OutputJob, OutputJob>(
async(outputJob) => await DownloadPayloadBlockCallback(outputJob, cancellationToken));
var exportActionBlock = new TransformBlock <OutputJob, OutputJob>(
async(outputJob) => await ExportDataBlockCallback(outputJob, cancellationToken),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Concurrentcy,
MaxMessagesPerTask = 1,
CancellationToken = cancellationToken
});
var reportingActionBlock = new ActionBlock <OutputJob>(
async(outputJob) => await ReportingActionBlock(outputJob, cancellationToken));
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
downloadActionBlock.LinkTo(dataConvertTransformBlock, linkOptions);
dataConvertTransformBlock.LinkTo(downloadPayloadTransformBlock, linkOptions);
downloadPayloadTransformBlock.LinkTo(exportActionBlock, linkOptions);
exportActionBlock.LinkTo(reportingActionBlock, linkOptions);
try
{
downloadActionBlock.Post(Agent);
downloadActionBlock.Complete();
reportingActionBlock.Completion.Wait();
_logger.Log(LogLevel.Trace, "Export Service completed timer routine.");
}
catch (AggregateException ex)
{
foreach (var iex in ex.InnerExceptions)
{
_logger.Log(LogLevel.Error, iex, "Error occurred while exporting.");
}
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, ex, "Error processing export task.");
}
finally
{
_workerTimer?.Start();
}
}
private static async Task DownloadAsync(string tfsUrl, string destinationFolder, string scope, Func <TfvcItem, bool> filter)
{
int maxConcurrency = 8;
ServicePointManager.DefaultConnectionLimit = maxConcurrency;
var baseUrl = new Uri(tfsUrl);
VssClientCredentials vssClientCredentials = new VssClientCredentials();
vssClientCredentials.Storage = new VssClientCredentialStorage();
var vssHttpRequestSettings = new VssHttpRequestSettings();
vssHttpRequestSettings.SendTimeout = TimeSpan.FromMilliseconds(-1);
var client = new TfvcHttpClient(baseUrl, vssClientCredentials, vssHttpRequestSettings);
try
{
var items = await client.GetItemsAsync(scopePath : scope, recursionLevel : VersionControlRecursionType.Full, includeLinks : false).ConfigureAwait(false);
var files = items.Where(filter).OrderBy(_ => _.Path).ToList();
var transformBlock = new TransformBlock <TfvcItem, TfvcItem>(async item =>
{
if (item.IsFolder)
{
var fullPath = GetFullPath(destinationFolder, item.Path);
if (!Directory.Exists(fullPath))
{
Directory.CreateDirectory(fullPath);
}
}
else
{
var fullPath = GetFullPath(destinationFolder, item.Path);
var folderPath = Path.GetDirectoryName(fullPath);
if (folderPath != null && !Directory.Exists(folderPath))
{
Directory.CreateDirectory(folderPath);
}
using (var stream = await client.GetItemContentAsync(item.Path))
using (var fs = File.Create(fullPath))
{
await stream.CopyToAsync(fs).ConfigureAwait(false);
}
}
return(item);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxConcurrency
});
var writePathBlock = new ActionBlock <TfvcItem>(c =>
{
var index = files.IndexOf(c);
Console.WriteLine($"{index}/{files.Count}: {c.Path}");
});
transformBlock.LinkTo(writePathBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (var item in files)
{
transformBlock.Post(item);
}
transformBlock.Complete();
await transformBlock.Completion.ConfigureAwait(false);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
public async Task Get([FromQuery] string text)
{
// Make sure that the length is limited to 1000 characters
text = text ?? string.Empty;
if (text.Length > 1000)
{
text = text.Substring(0, 1000);
}
var getResultBlock = new TransformBlock <MarkdownEntry, JObject>(async implem =>
{
var clock = Stopwatch.StartNew();
JObject jobject;
try
{
if (implem.Url == "js:commonmark.js")
{
jobject = await GetCommonMarkJs(text);
}
else if (implem.Url == "js:markdown-it")
{
jobject = await GetMarkdownIt(text);
}
else if (implem.POST)
{
var content = new ByteArrayContent(Encoding.UTF8.GetBytes(text));
content.Headers.Add("Content-Type", "text/plain");
var responseMessage =
await _httpClient.PostAsync(implem.Url, content);
jobject = new JObject
{
["html"] = await responseMessage.Content.ReadAsStringAsync()
};
if (!string.IsNullOrWhiteSpace(implem.VersionHeader))
{
jobject["version"] = responseMessage.Headers.GetValues(implem.VersionHeader).FirstOrDefault();
}
}
else
{
var jsonText =
await _httpClient.GetStringAsync(implem.Url + "text=" + Uri.EscapeDataString(text));
jobject = JObject.Parse(jsonText);
}
var html = jobject["html"]?.ToString() ?? string.Empty;
if (string.IsNullOrWhiteSpace(html))
{
html = string.Empty;
jobject["html_clean"] = html;
jobject["html_safe"] = html;
}
else
{
// Generates also a clean html in order to compare implems
var settings = HtmlSettings.Pretty();
settings.IsFragmentOnly = true;
settings.MinifyCss = false;
settings.MinifyCssAttributes = false;
settings.MinifyJs = false;
var result = Uglify.Html(html, settings);
jobject["html_clean"] = result.Code;
// Remove any javascript
settings.RemoveJavaScript = true;
result = Uglify.Html(html, settings);
jobject["html_safe"] = result.Code;
}
}
catch (Exception exception)
{
_logger.LogError("Unexpected exception: " + exception);
// In case we have an error, we still return an object
jobject = new JObject
{
["version"] = "unknown",
["error"] = GetPrettyMessageFromException(exception)
};
}
clock.Stop();
// Set common fields
jobject["name"] = implem.Name; // use the name from the registry, not the one returned
jobject["repo"] = implem.Repo;
jobject["cmark"] = implem.CommonMark;
jobject["lang"] = implem.Lang;
jobject["time"] = clock.Elapsed.TotalSeconds;
return(jobject);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
var returnResultBlock = new ActionBlock <JObject>(async jobject =>
{
var textWriter = new StringWriter();
var writer = new JsonTextWriter(textWriter)
{
Formatting = Formatting.None
};
jobject.WriteTo(writer);
textWriter.Write("\n\n");
var buffer = Encoding.UTF8.GetBytes(textWriter.ToString());
await HttpContext.Response.Body.WriteAsync(buffer, 0, buffer.Length);
await HttpContext.Response.Body.FlushAsync();
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
getResultBlock.LinkTo(returnResultBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
try
{
var entries = await MarkdownRegistry.Instance.GetEntriesAsync();
// We shuffle the entries to random the order of the latency of the results
Shuffle(entries);
foreach (var entry in entries)
{
await getResultBlock.SendAsync(entry);
}
getResultBlock.Complete();
await returnResultBlock.Completion;
}
catch (Exception ex)
{
_logger.LogError("Unexpected exception while fetching/returning: " + ex);
}
}
public async Task HandleAsync(CommandContext context, NextDelegate next)
{
if (context.Command is BulkUpdateAssets bulkUpdates)
{
if (bulkUpdates.Jobs?.Length > 0)
{
var executionOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2)
};
var createCommandsBlock = new TransformBlock <BulkTask, BulkTaskCommand?>(task =>
{
try
{
return(CreateCommand(task));
}
catch (OperationCanceledException ex)
{
// Dataflow swallows operation cancelled exception.
throw new AggregateException(ex);
}
}, executionOptions);
var executeCommandBlock = new ActionBlock <BulkTaskCommand?>(async command =>
{
try
{
if (command != null)
{
await ExecuteCommandAsync(command);
}
}
catch (OperationCanceledException ex)
{
// Dataflow swallows operation cancelled exception.
throw new AggregateException(ex);
}
}, executionOptions);
createCommandsBlock.BidirectionalLinkTo(executeCommandBlock);
contextProvider.Context.Change(b => b
.WithoutAssetEnrichment()
.WithoutCleanup()
.WithUnpublished(true)
.WithoutTotal());
var results = new ConcurrentBag <BulkUpdateResultItem>();
for (var i = 0; i < bulkUpdates.Jobs.Length; i++)
{
var task = new BulkTask(
context.CommandBus,
i,
bulkUpdates.Jobs[i],
bulkUpdates,
results);
if (!await createCommandsBlock.SendAsync(task))
{
break;
}
}
createCommandsBlock.Complete();
await executeCommandBlock.Completion;
context.Complete(new BulkUpdateResult(results));
}
else
{
context.Complete(new BulkUpdateResult());
}
}
else
{
await next(context);
}
}
static void Main(string[] args)
{
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock<string, string>(uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return new WebClient().DownloadString(uri);
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock<string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character to.
char[] tokens = text.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
tokens[i] = ' ';
}
text = new string(tokens);
// Separate the text into an array of words.
return text.Split(new char[] { ' ' },
StringSplitOptions.RemoveEmptyEntries);
});
// Removes short words, orders the resulting words alphabetically,
// and then remove duplicates.
var filterWordList = new TransformBlock<string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return words.Where(word => word.Length > 3).OrderBy(word => word)
.Distinct().ToArray();
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock<string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
// Holds reversed words.
var reversedWords = new ConcurrentQueue<string>();
// Add each word in the original collection to the result whose
// reversed word also exists in the collection.
Parallel.ForEach(words, word =>
{
// Reverse the work.
string reverse = new string(word.Reverse().ToArray());
// Enqueue the word if the reversed version also exists
// in the collection.
if (Array.BinarySearch<string>(words, reverse) >= 0 &&
word != reverse)
{
reversedWords.Enqueue(word);
}
});
return reversedWords;
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock<string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
//
// Connect the dataflow blocks to form a pipeline.
//
downloadString.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(findReversedWords);
findReversedWords.LinkTo(printReversedWords);
//
// For each completion task in the pipeline, create a continuation task
// that marks the next block in the pipeline as completed.
// A completed dataflow block processes any buffered elements, but does
// not accept new elements.
//
downloadString.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)createWordList).Fault(t.Exception);
else createWordList.Complete();
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterWordList).Fault(t.Exception);
else filterWordList.Complete();
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)findReversedWords).Fault(t.Exception);
else findReversedWords.Complete();
});
findReversedWords.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)printReversedWords).Fault(t.Exception);
else printReversedWords.Complete();
});
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
Console.ReadLine();
}
public void Stop()
{
_transformBlock.Complete();
}
static void Main(string[] args)
{
//
// Create members of the Pipeline
//
// Step 1 - Download the requested resource as a string
var downloadString = new TransformBlock <string, string>
(url =>
{
Console.WriteLine($"STEP 1 - Downloading from {url}...");
string result = null;
using (var client = new HttpClient())
{
// Perform a synchronous call by calling .Result
var response = client.GetAsync(url).Result;
if (response.IsSuccessStatusCode)
{
var responseContent = response.Content;
// read result synchronously by calling .Result
result = responseContent.ReadAsStringAsync().Result;
if (!string.IsNullOrEmpty(result))
{
Console.WriteLine($"STEP 1 - Downloaded {result.Length} characters...");
}
}
}
return(result);
}
);
// Step 2 - Separate the specified text into an array of words
var createWordList = new TransformBlock <string, List <string> >
(text =>
{
Console.WriteLine("STEP 2 - Creating word list...");
char[] tokens = text.ToCharArray();
// replace non-letter chars like punctuations with space char.
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
{
tokens[i] = ' ';
}
}
text = new string(tokens);
var stringList = text.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries).ToList();
Console.WriteLine($"STEP 2 - Found {stringList.Count} words...");
return(stringList);
}
);
// Step 3 - Remove short words, sort alphabetically and remove dupes.
var filterWordList = new TransformBlock <List <string>, List <string> >
(wordsList =>
{
Console.WriteLine("STEP 3 - Filtering word list...");
var filteredWordsList = wordsList.Where(word => word.Length > 3).OrderBy(word => word).Distinct().ToList();
Console.WriteLine($"STEP 3 - Filtered list down to {filteredWordsList.Count}...");
return(filteredWordsList);
}
);
// Step 4 - Find Palindromes.
var findPalindromeWords = new TransformManyBlock <List <string>, string>
(wordsList =>
{
Console.WriteLine("STEP 4 - Finding Palindrome words...");
// Holds palindrome words.
var palindromeWords = new ConcurrentQueue <string>();
// Add each word in the original collection to the result whose
// reversed word also exists in the collection.
Parallel.ForEach(wordsList, word =>
{
// Reverse the word.
string reverse = new string(word.Reverse().ToArray());
// Enqueue the word if the reversed version also exists
// in the collection.
if (word == reverse)
{
palindromeWords.Enqueue(word);
}
});
var searchMessage = palindromeWords.Any() ? $"STEP 4 - Found {palindromeWords.Count} palindrome(s)" : $"STEP 4 - Didn't find any palindromes :(";
Console.WriteLine(searchMessage);
return(palindromeWords);
});
// Step 5 - Print Palindromes.
var printPalindromes = new ActionBlock <string>
(palindrome =>
{
Console.WriteLine($"STEP 5 - Found palindrome: {palindrome} / {new string(palindrome.Reverse().ToArray())}");
}
);
downloadString.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(findPalindromeWords);
findPalindromeWords.LinkTo(printPalindromes);
downloadString.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)createWordList).Fault(t.Exception);
}
else
{
createWordList.Complete();
}
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)filterWordList).Fault(t.Exception);
}
else
{
filterWordList.Complete();
}
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)findPalindromeWords).Fault(t.Exception);
}
else
{
findPalindromeWords.Complete();
}
});
findPalindromeWords.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)printPalindromes).Fault(t.Exception);
}
else
{
printPalindromes.Complete();
}
});
// Process "Adventures of Huckleberry Finn"
// by Mark Twain
Console.WriteLine("STEP 0 - Processing Mark Twain's Book - Adventures of Huckleberry Finn");
downloadString.Post("http://www.gutenberg.org/files/76/76-0.txt");
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printPalindromes.Completion.Wait();
Console.WriteLine("Done!\nPress a key to exit:");
Console.ReadKey();
}
public async Task DecompressEntryAsync(int index, Stream output)
{
var entry = Files[index];
using (var fs = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.None, 4096, useAsync: true))
{
fs.Seek(entry.BlockOffsets[0], SeekOrigin.Begin);
//not compressed
if (entry.BlockSizeIndex == 0xFFFFFFFF)
{
var uncompressed = new byte[entry.RealUncompressedSize];
await fs.ReadAsync(uncompressed, 0, uncompressed.Length).ConfigureAwait(continueOnCapturedContext: false);
await output.WriteAsync(uncompressed, 0, uncompressed.Length).ConfigureAwait(continueOnCapturedContext: false);
return;
}
var decompressor = new TransformBlock <InputBlock, byte[]>(
input => input.IsCompressed
? SevenZipHelper.Decompress(input.Data, input.UncompressedSize)
: input.Data
, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
}
);
var outputWriter = new ActionBlock <byte[]>(
data => output.Write(data, 0, data.Length)
);
decompressor.LinkTo(outputWriter, new DataflowLinkOptions {
PropagateCompletion = true
});
uint count = 0;
long left = entry.RealUncompressedSize;
while (left > 0)
{
uint compressedBlockSize = entry.BlockSizes[count];
if (compressedBlockSize == 0)
{
compressedBlockSize = Header.MaxBlockSize;
}
if (compressedBlockSize == Header.MaxBlockSize ||
compressedBlockSize == left)
{
left -= compressedBlockSize;
var uncompressedData = new byte[compressedBlockSize];
await fs.ReadAsync(uncompressedData, 0, uncompressedData.Length).ConfigureAwait(continueOnCapturedContext: false);
decompressor.Post(new InputBlock(uncompressedData, InputBlock.Uncompressed));
}
else
{
var uncompressedBlockSize = Math.Min(left, Header.MaxBlockSize);
left -= uncompressedBlockSize;
if (compressedBlockSize < 5)
{
throw new Exception("compressed block size smaller than 5");
}
var compressedData = new byte[compressedBlockSize];
await fs.ReadAsync(compressedData, 0, (int)compressedBlockSize).ConfigureAwait(continueOnCapturedContext: false);
decompressor.Post(new InputBlock(compressedData, uncompressedBlockSize));
}
count++;
}
decompressor.Complete();
await outputWriter.Completion;
}
}
public void RunTransformBlockConformanceTests()
{
bool passed = true;
// SYNC
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++)
{
Func <DataflowBlockOptions, TargetProperties <int> > transformBlockFactory =
options =>
{
TransformBlock <int, int> transformBlock = new TransformBlock <int, int>(i => i, (ExecutionDataflowBlockOptions)options);
ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformBlock.LinkTo(actionBlock);
return(new TargetProperties <int> {
Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false
});
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
var mptOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2
};
var cancellationOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token
};
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, dopOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, cancellationOptions, 1, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformBlock <int, int>, int>(4, () => new TransformBlock <int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformBlock <int, int>, int>(4, () => new TransformBlock <int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformBlock <int, int>, int>(4, () => new TransformBlock <int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++)
{
localPassed &= network.Post(i) == true;
}
for (int i = 0; i < ITERS; i++)
{
localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16;
}
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformBlock <int, int>, int>(4, () => new TransformBlock <int, int>(i => i * 2));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++)
{
tasks[i - 1] = network.SendAsync(i);
}
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++)
{
total += ((IReceivableSourceBlock <int>)network).Receive();
}
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformBlock <int, int>(i =>
{
if ((i % 2) == 0)
{
throw new OperationCanceledException();
}
return(i);
});
for (int i = 0; i < 2; i++)
{
t.Post(i);
}
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0)
{
localPassed &= t.Receive() == i;
}
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions {
CancellationToken = cts.Token
};
var t = new TransformBlock <int, int>(i => i, dbo);
int ignoredValue;
IList <int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock <int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformBlock <int, int>(new Func <int, int>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
// ASYNC (a copy of the sync but with constructors returning Task<T> instead of T
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++)
{
Func <DataflowBlockOptions, TargetProperties <int> > transformBlockFactory =
options =>
{
TransformBlock <int, int> transformBlock = new TransformBlock <int, int>(i => Task.Factory.StartNew(() => i), (ExecutionDataflowBlockOptions)options);
ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformBlock.LinkTo(actionBlock);
return(new TargetProperties <int> {
Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false
});
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
var mptOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2
};
var cancellationOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token
};
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, dopOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformBlock <int, int>, int>(4, () => new TransformBlock <int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformBlock <int, int>, int>(4, () => new TransformBlock <int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformBlock <int, int>, int>(4, () => new TransformBlock <int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++)
{
localPassed &= network.Post(i) == true;
}
for (int i = 0; i < ITERS; i++)
{
localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16;
}
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformBlock <int, int>, int>(4, () => new TransformBlock <int, int>(i => Task.Factory.StartNew(() => i * 2)));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++)
{
tasks[i - 1] = network.SendAsync(i);
}
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++)
{
total += ((IReceivableSourceBlock <int>)network).Receive();
}
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformBlock <int, int>(i =>
{
if ((i % 2) == 0)
{
throw new OperationCanceledException();
}
return(Task.Factory.StartNew(() => i));
});
for (int i = 0; i < 2; i++)
{
t.Post(i);
}
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0)
{
localPassed &= t.Receive() == i;
}
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformBlock <int, int>(i =>
{
if ((i % 2) == 0)
{
return(null);
}
return(Task.Factory.StartNew(() => i));
});
for (int i = 0; i < 2; i++)
{
t.Post(i);
}
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0)
{
localPassed &= t.Receive() == i;
}
}
t.Completion.Wait();
Console.WriteLine("{0}: null tasks are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformBlock <int, int>(i =>
{
if (i == 0)
{
Task.Delay(10).Wait();
return(null);
}
return(Task.Factory.StartNew(() => i));
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 2
});
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformBlock <int, int>(new Func <int, Task <int> >(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformBlock <int, int>(new Func <int, Task <int> >(i => Task <int> .Factory.StartNew(() => { throw new InvalidOperationException(); })));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
public async Task <IEnumerable <DummyData> > LoadAsync(IEnumerable <Uri> uris)
{
IList <DummyData> result;
using (var client = new HttpClient())
{
// downloader block with parallelism limit
var downloader = new TransformBlock <Uri, HttpResponseMessage>(
async u => await client.GetAsync(u),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = _maxParallelism,
EnsureOrdered = false,
SingleProducerConstrained = true
});
// deserializer, unbound parallelism
var deserializer =
new TransformBlock <HttpResponseMessage, DummyData>(
async r =>
{
using (Stream s = await r.Content.ReadAsStreamAsync())
using (var sr = new StreamReader(s))
using (JsonReader reader = new JsonTextReader(sr))
{
return(Serializer.Deserialize <DummyData>(reader));
}
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded,
EnsureOrdered = false,
SingleProducerConstrained = true
});
// buffer to access result
var buffer = new BufferBlock <DummyData>(
new ExecutionDataflowBlockOptions
{
EnsureOrdered = false
});
// link blocks together
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
downloader.LinkTo(deserializer, linkOptions);
deserializer.LinkTo(buffer, linkOptions);
// start sending input
foreach (Uri uri in uris)
{
await downloader.SendAsync(uri);
}
// input completed
downloader.Complete();
// await deserializer
await deserializer.Completion;
// pipeline done, receive result
buffer.TryReceiveAll(out result);
}
return(result);
}
static void Main(string[] args)
{
int fileCounter = 0;
string outputDirectory = ".";
TestClassGenerator generator = new TestClassGenerator();
int maxLoading = 0;
var downloadString = new TransformBlock <string, Task <string> >(path =>
{
while (fileLoadingCounter >= maxFileLoading)
{
;
}
StreamReader sr = new StreamReader(path);
Task <string> result = sr.ReadToEndAsync();
fileLoadingCounter++;
if (maxLoading < fileLoadingCounter)
{
maxLoading = fileLoadingCounter;
}
result.ContinueWith(str => { fileLoadingCounter--; });
return(result);
});
var generateClass = new TransformBlock <Task <string>, Task <string> >(async sourceTask =>
{
while (testsGeneratingCounter >= maxTestsGenerating)
{
;
}
testsGeneratingCounter++;
Task <string> task = generator.GenerateTestAsync(await sourceTask);
Task temp = task.ContinueWith(str => { testsGeneratingCounter--; });
return(task);
});
var outputString = new ActionBlock <Task <string> >(async sourceTask =>
{
string source = await sourceTask;
while (fileWritingCounter >= maxFileWriting)
{
;
}
fileWritingCounter++;
fileCounter++;
StreamWriter sw = new StreamWriter($"{outputDirectory}/Test{fileCounter}.cs");
Task t = sw.WriteAsync(source);
Task temp = t.ContinueWith((str) => { fileWritingCounter--; });
t.Wait();
sw.Close();
});
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
downloadString.LinkTo(generateClass, linkOptions);
generateClass.LinkTo(outputString, linkOptions);
List <string> fileNames = new List <string>()
{
"Program.cs", "AssemblyGetter.cs",
"StudentExtension.cs", "Factorizer.cs"
};
foreach (var name in fileNames)
{
downloadString.Post(name);
}
downloadString.Complete();
outputString.Completion.Wait();
Console.WriteLine(maxLoading);
Console.ReadLine();
}
public async Task PostList(ITargetBlock<Core.Entities.BalanceDate> target)
{
Diag.ThreadPrint("PostList - start");
var transform = new TransformBlock<BalanceDate, Core.Entities.BalanceDate>(ef =>
mapper.Map<Core.Entities.BalanceDate>(ef), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 });
transform.LinkTo(target,new DataflowLinkOptions() { PropagateCompletion = true });
await Task.Run(() =>
{
Diag.ThreadPrint("PostList - task start");
using (FinanceEntities context = factory.CreateContext())
{
(from b in context.BalanceDates
.Include(a => a.BalanceDateBankAccounts)
.Include("BalanceDateBankAccounts.BankAccount")
.Include("BalanceDateBankAccounts.BankAccount.Bank")
select b).AsParallel().ForAll(ef => transform.Post(ef));
//await transform.Completion;
//transform.Completion.ContinueWith(t =>
//{
// if (t.IsFaulted) target.Fault(t.Exception);
// else
// {
// Diag.ThreadPrint("PostList - task set target complete");
// target.Complete();
// }
//});
transform.Complete();
}
Diag.ThreadPrint("PostList - task end");
}).ConfigureAwait(false);
Diag.ThreadPrint("PostList - end");
}
public async Task TestOrdering()
{
const int iters = 1000;
foreach (int mmpt in new[] { DataflowBlockOptions.Unbounded, 1 })
foreach (int dop in new[] { 1, 2, DataflowBlockOptions.Unbounded })
{
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, MaxMessagesPerTask = mmpt };
var tb = new TransformBlock<int, int>(i => i, options);
tb.PostRange(0, iters);
for (int i = 0; i < iters; i++)
{
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
}
tb.Complete();
await tb.Completion;
}
}
public Task RunAsync()
{
TransformBlock <Uri, PropertyBag> ingestBlock = new TransformBlock <Uri, PropertyBag>(input =>
{
PropertyBag result = new PropertyBag
{
OriginalUrl = input.ToString(),
UserAgent = _userAgent,
Step = new CrawlStep(input, 0)
};
return(result);
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelism
});
TransformBlock <PropertyBag, PropertyBag> ingestBlockForAggregation =
new TransformBlock <PropertyBag, PropertyBag>(input => input, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelism
});
CrawlIngestionHelper crawlIngestionHelper = new CrawlIngestionHelper(ingestBlockForAggregation, _userAgent);
TransformBlock <PropertyBag, PropertyBag>[] pipeline = Pipeline
.Select(pipelineStep =>
{
return(new TransformBlock <PropertyBag, PropertyBag>(async propertyBag =>
{
if (propertyBag.StopPipelining)
{
return propertyBag;
}
try
{
propertyBag.StopPipelining = !await pipelineStep.Process(crawlIngestionHelper, propertyBag);
}
catch (Exception exception)
{
propertyBag.Exceptions.Add(exception);
}
return propertyBag;
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = pipelineStep.MaxDegreeOfParallelism
}));
})
.ToArray();
ActionBlock <PropertyBag> terminationCheckerBlock = new ActionBlock <PropertyBag>(propertyBag =>
{
if (ingestBlock.InputCount == 0 &&
ingestBlock.OutputCount == 0 &&
!ingestBlock.Completion.IsCompleted &&
!ingestBlock.Completion.IsCanceled &&
!ingestBlock.Completion.IsFaulted &&
ingestBlockForAggregation.InputCount == 0 &&
ingestBlockForAggregation.OutputCount == 0)
{
if (pipeline.Any(transformBlock => transformBlock.InputCount != 0 || transformBlock.OutputCount != 0))
{
return;
}
ingestBlock.Complete();
}
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
});
ingestBlock.LinkTo(ingestBlockForAggregation, new DataflowLinkOptions {
PropagateCompletion = true
});
TransformBlock <PropertyBag, PropertyBag> previous = ingestBlockForAggregation;
foreach (TransformBlock <PropertyBag, PropertyBag> transformBlock in pipeline)
{
previous.LinkTo(transformBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
previous = transformBlock;
}
previous.LinkTo(terminationCheckerBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (Uri startUri in StartUris)
{
ingestBlock.Post(startUri);
}
return(terminationCheckerBlock.Completion);
}
public void AddFiles(IEnumerable<FileInfo> files)
{
var tb = new TransformBlock<FileInfo, FileInfo>(file =>
{
if (!TransferredFiles.Contains(file))
return file;
return null;
});
var ab = new ActionBlock<FileInfo>(file =>
{
if (file != null)
DetectedFiles.Push(file);
});
Parallel.ForEach(files, async file => await tb.SendAsync(file));
tb.LinkTo(ab);
tb.Complete();
tb.Completion.Wait();
//
// Save Files
//
Task.Run(async () => await TransformPhysicalDisk.SecureDataSaverAsync(DetectedFiles.ToString(), DetectedData_Path, HashingPass));
}
public async Task StartPipelineWithBackPressureAsync(CancellationToken token)
{
_decoder.LoadSensorConfigs();
// Step 1 - Configure the pipeline
// make sure our complete call gets propagated throughout the whole pipeline
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
// create our block configurations
var largeBufferOptions = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 60000
};
var smallBufferOptions = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1000
};
var parallelizedOptions = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1000, MaxDegreeOfParallelism = 4
};
var batchOptions = new GroupingDataflowBlockOptions()
{
BoundedCapacity = 200000
};
// create some branching functions for our TransformManyBlocks
// DecodedMessage gets tranformed into 3 RoutedMessage for the first three way branch
Func <DecodedMessage, IEnumerable <RoutedMessage> > messageBranchFunc = x => new List <RoutedMessage>
{
new RoutedMessage(1, x),
new RoutedMessage(2, x),
new RoutedMessage(3, x)
};
// DecodedMessage[] gets tranformed into a RoutedBatch for the final branch
Func <RoutedMessage[], IEnumerable <RoutedBatch> > batchBranchFunc = x => new List <RoutedBatch>
{
new RoutedBatch(1, x.Select(c => c.Message).ToList()),
new RoutedBatch(2, x.Select(c => c.Message).ToList())
};
// define each block
var writeRawMessageBlock = new TransformBlock <RawBusMessage, RawBusMessage>(async(RawBusMessage msg) =>
{
await _messageFileWriter.WriteAsync(msg);
return(msg);
}, largeBufferOptions);
var decoderBlock = new TransformManyBlock <RawBusMessage, DecodedMessage>(
(RawBusMessage msg) => _decoder.Decode(msg), largeBufferOptions);
var msgBranchBlock = new TransformManyBlock <DecodedMessage, RoutedMessage>(messageBranchFunc, largeBufferOptions);
var realTimeFeedBlock = new ActionBlock <RoutedMessage>(async(RoutedMessage routedMsg) =>
await _realTimeFeedPublisher.PublishAsync(routedMsg.Message), largeBufferOptions);
var thirtySecondBatchBlock = new BatchBlock <RoutedMessage>(90000, batchOptions);
var thirtySecondStatsFeedBlock = new ActionBlock <RoutedMessage[]>(async(RoutedMessage[] batch) =>
await _statsFeedPublisher.PublishAsync(batch.Select(x => x.Message).ToList(), TimeSpan.FromSeconds(30)), smallBufferOptions);
var oneSecondBatchBlock = new BatchBlock <RoutedMessage>(3000, batchOptions);
var batchBroadcastBlock = new TransformManyBlock <RoutedMessage[], RoutedBatch>(batchBranchFunc, smallBufferOptions);
var oneSecondStatsFeedBlock = new ActionBlock <RoutedBatch>(async(RoutedBatch batch) =>
await _statsFeedPublisher.PublishAsync(batch.Messages.ToList(), TimeSpan.FromSeconds(1)), smallBufferOptions);
var dbPersistenceBlock = new ActionBlock <RoutedBatch>(async(RoutedBatch batch) =>
await _dbPersister.PersistAsync(batch.Messages.ToList()), parallelizedOptions);
// link the blocks to together
writeRawMessageBlock.LinkTo(decoderBlock, linkOptions);
decoderBlock.LinkTo(msgBranchBlock, linkOptions);
msgBranchBlock.LinkTo(realTimeFeedBlock, linkOptions, routedMsg => routedMsg.RouteKey == 1); // route on the key
msgBranchBlock.LinkTo(oneSecondBatchBlock, linkOptions, routedMsg => routedMsg.RouteKey == 2); // route on the key
msgBranchBlock.LinkTo(thirtySecondBatchBlock, linkOptions, routedMsg => routedMsg.RouteKey == 3); // route on the key
thirtySecondBatchBlock.LinkTo(thirtySecondStatsFeedBlock, linkOptions);
oneSecondBatchBlock.LinkTo(batchBroadcastBlock, linkOptions);
batchBroadcastBlock.LinkTo(oneSecondStatsFeedBlock, linkOptions, routedMsg => routedMsg.RouteKey == 1); // route on the key
batchBroadcastBlock.LinkTo(dbPersistenceBlock, linkOptions, routedMsg => routedMsg.RouteKey == 2); // route on the key
// Step 2 - Start consuming the machine bus interface (the producer)
var consumerTask = _dataBusReader.StartConsuming(writeRawMessageBlock, token, TimeSpan.FromMilliseconds(1000), FlowControlMode.BackPressure);
// Step 3 - Keep going until the CancellationToken is cancelled or a leaf block is the the completed state either due to a fault or the completion of the pipeline.
while (!token.IsCancellationRequested ||
oneSecondStatsFeedBlock.Completion.IsCompleted ||
dbPersistenceBlock.Completion.IsCompleted ||
realTimeFeedBlock.Completion.IsCompleted ||
thirtySecondStatsFeedBlock.Completion.IsCompleted)
{
await Task.Delay(500);
}
// Step 4 - the CancellationToken has been cancelled and our producer has stopped producing
// call Complete on the first block, this will propagate down the pipeline
writeRawMessageBlock.Complete();
// wait for all leaf blocks to finish processing their data
await Task.WhenAll(oneSecondStatsFeedBlock.Completion,
thirtySecondStatsFeedBlock.Completion,
dbPersistenceBlock.Completion,
realTimeFeedBlock.Completion,
consumerTask);
// clean up any other resources like ZeroMQ for example
}
public async Task TestOrdering_Async_OrderedDisabled()
{
// If ordering were enabled, this test would hang.
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded, EnsureOrdered = false };
var tasks = new TaskCompletionSource<int>[10];
for (int i = 0; i < tasks.Length; i++)
{
tasks[i] = new TaskCompletionSource<int>();
}
var tb = new TransformBlock<int, int>(i => tasks[i].Task, options);
tb.PostRange(0, tasks.Length);
for (int i = tasks.Length - 1; i >= 0; i--)
{
tasks[i].SetResult(i);
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
}
tb.Complete();
await tb.Completion;
}
public async Task StartPipelineAsync(CancellationToken token)
{
_decoder.LoadSensorConfigs();
// Step 1 - Configure the pipeline
// make sure our complete call gets propagated throughout the whole pipeline
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
// create our block configurations
var largeBufferOptions = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 600000
};
var smallBufferOptions = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1000
};
var realTimeBufferOptions = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 6000
};
var parallelizedOptions = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1000, MaxDegreeOfParallelism = 4
};
var batchOptions = new GroupingDataflowBlockOptions()
{
BoundedCapacity = 1000
};
// define each block
var writeRawMessageBlock = new TransformBlock <RawBusMessage, RawBusMessage>(async(RawBusMessage msg) =>
{
await _messageFileWriter.WriteAsync(msg);
return(msg);
}, largeBufferOptions);
var decoderBlock = new TransformManyBlock <RawBusMessage, DecodedMessage>(
(RawBusMessage msg) => _decoder.Decode(msg), largeBufferOptions);
var broadcast = new BroadcastBlock <DecodedMessage>(msg => msg);
var realTimeFeedBlock = new ActionBlock <DecodedMessage>(async
(DecodedMessage msg) => await _realTimeFeedPublisher.PublishAsync(msg), realTimeBufferOptions);
var oneSecondBatchBlock = new BatchBlock <DecodedMessage>(3000);
var thirtySecondBatchBlock = new BatchBlock <DecodedMessage>(90000);
var batchBroadcastBlock = new BroadcastBlock <DecodedMessage[]>(msg => msg);
var oneSecondStatsFeedBlock = new ActionBlock <DecodedMessage[]>(async
(DecodedMessage[] messages) => await _statsFeedPublisher.PublishAsync(messages.ToList(), TimeSpan.FromSeconds(1)), smallBufferOptions);
var dbPersistenceBlock = new ActionBlock <DecodedMessage[]>(async
(DecodedMessage[] messages) => await _dbPersister.PersistAsync(messages.ToList()), smallBufferOptions);
var thirtySecondStatsFeedBlock = new ActionBlock <DecodedMessage[]>(async
(DecodedMessage[] messages) => await _statsFeedPublisher.PublishAsync(messages.ToList(), TimeSpan.FromSeconds(30)), smallBufferOptions);
// link the blocks to together
writeRawMessageBlock.LinkTo(decoderBlock, linkOptions);
decoderBlock.LinkTo(broadcast, linkOptions);
broadcast.LinkTo(realTimeFeedBlock, linkOptions);
broadcast.LinkTo(oneSecondBatchBlock, linkOptions);
broadcast.LinkTo(thirtySecondBatchBlock, linkOptions);
oneSecondBatchBlock.LinkTo(batchBroadcastBlock, linkOptions);
batchBroadcastBlock.LinkTo(oneSecondStatsFeedBlock, linkOptions);
batchBroadcastBlock.LinkTo(dbPersistenceBlock, linkOptions);
thirtySecondBatchBlock.LinkTo(thirtySecondStatsFeedBlock, linkOptions);
// Step 2 - Start consuming the machine bus interface (the producer)
var consumerTask = _dataBusReader.StartConsuming(writeRawMessageBlock, token, TimeSpan.FromMilliseconds(1000), FlowControlMode.LoadShed);
// Step 3 - Keep going until the CancellationToken is cancelled or a leaf block is the the completed state either due to a fault or the completion of the pipeline.
while (!token.IsCancellationRequested &&
!realTimeFeedBlock.Completion.IsCompleted &&
!oneSecondStatsFeedBlock.Completion.IsCompleted &&
!dbPersistenceBlock.Completion.IsCompleted &&
!thirtySecondStatsFeedBlock.Completion.IsCompleted)
{
await Task.Delay(500);
}
// Step 4 - the CancellationToken has been cancelled and our producer has stopped producing
// call Complete on the first block, this will propagate down the pipeline
writeRawMessageBlock.Complete();
// wait for all leaf blocks to finish processing their data
await Task.WhenAll(realTimeFeedBlock.Completion,
oneSecondStatsFeedBlock.Completion,
dbPersistenceBlock.Completion,
thirtySecondStatsFeedBlock.Completion,
consumerTask);
// clean up any other resources like ZeroMQ for example
}
public string Name(IPost post)
{
// Keep tags separated. This is faster than enumerating
// over a large collection with all the tags using LINQ.
var artistTags = new List <ITag>();
var copyrightTags = new List <ITag>();
var characterTags = new List <ITag>();
// Put tags into their own lists.
var addTagBlock = new ActionBlock <ITag?>(tag =>
{
switch (tag?.Kind)
{
case TagKind.Artist:
artistTags.Add(tag);
break;
case TagKind.Copyright:
copyrightTags.Add(tag);
break;
case TagKind.Character:
characterTags.Add(tag);
break;
}
});
if (post is IPostExtendedTags postExtendedTags)
{
postExtendedTags.ExtendedTags.ForEach(tag => addTagBlock.Post(tag));
addTagBlock.Complete();
}
else
{
// Get tags using provided function in parallel.
var getTagBlock = new TransformBlock <string, ITag?>(
async tagName =>
{
try
{
return(await _tagExtractorFunc(tagName).CAF());
}
catch (InvalidTagNameException)
{
Logger.Debug($"Got invalid tag '{tagName}'.", this);
return(null);
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = MaxActiveTagRequestsCount
});
// IMPORTANT: getTagBlock must propagate its completion if we want to wait for
// addTagBlock's completion. This is the correct usage as per MS docs.
getTagBlock.LinkTo(addTagBlock, _linkOptions);
if (post is IPostExtraTags extraTags)
{
void postTag(string tag) => getTagBlock.Post(tag);
extraTags.CharacterTags.ForEach(postTag);
extraTags.CopyrightTags.ForEach(postTag);
extraTags.ArtistTags.ForEach(postTag);
}
else
{
foreach (string tag in post.Tags)
{
getTagBlock.Post(tag);
}
}
getTagBlock.Complete();
}
// Wait until all the tags are processed.
addTagBlock.Completion.Wait();
static void Main(string[] args)
{
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock <string, string>(uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return(new WebClient().DownloadString(uri));
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character to.
char[] tokens = text.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
{
tokens[i] = ' ';
}
}
text = new string(tokens);
// Separate the text into an array of words.
return(text.Split(new char[] { ' ' },
StringSplitOptions.RemoveEmptyEntries));
});
// Removes short words, orders the resulting words alphabetically,
// and then remove duplicates.
var filterWordList = new TransformBlock <string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return(words.Where(word => word.Length > 3).OrderBy(word => word)
.Distinct().ToArray());
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock <string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
// Holds reversed words.
var reversedWords = new ConcurrentQueue <string>();
// Add each word in the original collection to the result whose
// reversed word also exists in the collection.
Parallel.ForEach(words, word =>
{
// Reverse the work.
string reverse = new string(word.Reverse().ToArray());
// Enqueue the word if the reversed version also exists
// in the collection.
if (Array.BinarySearch <string>(words, reverse) >= 0 &&
word != reverse)
{
reversedWords.Enqueue(word);
}
});
return(reversedWords);
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock <string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
//
// Connect the dataflow blocks to form a pipeline.
//
downloadString.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(findReversedWords);
findReversedWords.LinkTo(printReversedWords);
//
// For each completion task in the pipeline, create a continuation task
// that marks the next block in the pipeline as completed.
// A completed dataflow block processes any buffered elements, but does
// not accept new elements.
//
downloadString.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)createWordList).Fault(t.Exception);
}
else
{
createWordList.Complete();
}
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)filterWordList).Fault(t.Exception);
}
else
{
filterWordList.Complete();
}
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)findReversedWords).Fault(t.Exception);
}
else
{
findReversedWords.Complete();
}
});
findReversedWords.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)printReversedWords).Fault(t.Exception);
}
else
{
printReversedWords.Complete();
}
});
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
Console.ReadLine();
}
internal static bool TransformThroughDiscardingFilterToAction()
{
const int ITERS = 2;
int completedCount = 0;
var t = new TransformBlock<int, int>(i => i);
var c = new ActionBlock<int>(i => completedCount++);
t.LinkTo(c, i => i % 2 == 0);
t.LinkTo(DataflowBlock.NullTarget<int>());
t.Completion.ContinueWith(_ => c.Complete());
for (int i = 0; i < ITERS; i++) t.Post(i);
t.Complete();
c.Completion.Wait();
return completedCount == ITERS / 2;
}
private void StartPipeline()
{
_fetcherBlock.Post("TestSupplier");
_fetcherBlock.Complete();
}
public static async Task DecompressFastDataFlow(Stream inputStream, Stream outputStream)
{
var buffer = new BufferBlock<DecompressionDetails>(new DataflowBlockOptions { BoundedCapacity = BoundedCapacity });
var compressorOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelism,
BoundedCapacity = BoundedCapacity
};
var writerOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = BoundedCapacity,
SingleProducerConstrained = true
};
var compressor = new TransformBlock<DecompressionDetails, DecompressionDetails>(compressionDetails => Decompress(compressionDetails), compressorOptions);
var writer = new ActionBlock<DecompressionDetails>(compressionDetailsWithSize => Multiplex(buffer, outputStream, compressionDetailsWithSize), writerOptions);
buffer.LinkTo(compressor);
compressor.LinkTo(writer);
buffer.Completion.ContinueWith(task => compressor.Complete());
compressor.Completion.ContinueWith(task => writer.Complete());
byte[] size = new byte[sizeof(long)];
await inputStream.ReadAsync(size, 0, size.Length);
// convert the size to number
long sourceLength = BitConverter.ToInt64(size, 0);
int index = 0;
while (sourceLength > 0)
{
size = new byte[sizeof(long)];
await inputStream.ReadAsync(size, 0, size.Length);
// convert the size back to number
long chunkSize = BitConverter.ToInt64(size, 0);
if (chunkSize > sourceLength) throw new InvalidDataException("");
// read the compressed size
size = new byte[sizeof(int)];
await inputStream.ReadAsync(size, 0, size.Length);
// convert the size back to number
int storedSize = BitConverter.ToInt32(size, 0);
byte[] compressedData = new byte[storedSize];
int readCount = await inputStream.ReadAsync(compressedData, 0, compressedData.Length);
DecompressionDetails decompressionDetails = new DecompressionDetails
{
Bytes = compressedData,
ChunkSize = chunkSize,
Sequence = ++index
};
while (await buffer.SendAsync(decompressionDetails) != true) { }
sourceLength -= chunkSize;
if (sourceLength < chunkSize)
chunkSize = sourceLength;
if (sourceLength == 0)
buffer.Complete();
}
writer.Completion.Wait();
outputStream.Flush();
inputStream.Dispose();
outputStream.Dispose();
}
public async Task <IEnumerable <string> > ProcessInParallel(IOutputWriter logListener, int maxDegreeOfParallelism = 8)
{
var theResults = new List <string>();
var processBlock = new TransformBlock <int, string>(async number =>
{
var processedNumber = await ProcessNumber(number);
return(processedNumber);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = maxDegreeOfParallelism,
BoundedCapacity = 10
});
var putInListBlock = new ActionBlock <string>(t =>
{
if (logListener != null)
{
logListener.WriteOutput(t);
}
theResults.Add(t);
}, SynchronizeForUiThread(new ExecutionDataflowBlockOptions()
{
}));
processBlock.LinkTo(putInListBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
var ttt = Task.Run(async() =>
{
while (true)
{
Console.WriteLine($">>>>> Threads: {currentThreadsRunning} In: {processBlock.InputCount} Out: {processBlock.OutputCount} >> In: {putInListBlock.InputCount} Out: {putInListBlock.Completion.IsCompleted}");
await Task.Delay(1000);
}
});
var numbers = EnumerateNumberList(100);
await Task.Run(async() =>
{
foreach (var number in numbers)
{
Console.WriteLine($"Posting: {number}");
var result = await processBlock.SendAsync(number);
if (!result)
{
Console.WriteLine("Result is false!!!");
}
}
});
Console.WriteLine("Completing");
processBlock.Complete();
await putInListBlock.Completion;
return(theResults);
}
private static async Task <UsageInformation> GetUsageInformation(IEnumerable <string> files)
{
var result = new UsageInformation();
var multipleInstancesOption = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
};
var singleInstanceOption = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
};
var getApisBlock = new TransformBlock <string, UsageInformation?>(GetApisUsedByAssembly, multipleInstancesOption);
var collectApisBlock = new ActionBlock <UsageInformation?>(assemblyInfo =>
{
if (assemblyInfo != null)
{
foreach (var targetFramework in assemblyInfo.TargetFrameworks)
{
result.TargetFrameworks.Add(targetFramework);
}
foreach (var version in assemblyInfo.Versions)
{
result.Versions.Add(version);
}
foreach (var apisByAssembly in assemblyInfo.MemberReferences)
{
if (!result.MemberReferences.TryGetValue(apisByAssembly.Key, out HashSet <string>?allApis))
{
allApis = new HashSet <string>();
result.MemberReferences[apisByAssembly.Key] = allApis;
}
foreach (var api in apisByAssembly.Value)
{
allApis.Add(api);
}
}
}
},
singleInstanceOption);
var linkOptions = new DataflowLinkOptions()
{
PropagateCompletion = true
};
getApisBlock.LinkTo(collectApisBlock, linkOptions);
foreach (var file in files)
{
await getApisBlock.SendAsync(file);
}
getApisBlock.Complete();
await Task.WhenAll(getApisBlock.Completion, collectApisBlock.Completion);
return(result);
}
public void Reset()
{
_builder?.Complete();
_builder = null;
}
public override void Execute()
{
SourceBufferBlock = new BufferBlock <string[]>();
DestinationBatchBlock = new BatchBlock <string[]>(BatchSize);
if (Source == null)
{
Source = new CSVSource(FileName);
}
using (Source) {
Source.Open();
Destination = new DBDestination()
{
Connection = DbConnectionManager, TableName = TableName
};
NLogger.Info(TaskName, TaskType, "START", TaskHash, ControlFlow.STAGE, ControlFlow.CurrentLoadProcess?.LoadProcessKey);
/* Pipeline:
* Source -> BufferBlock -> RowTransformation -> BatchBlock -> BatchTransformation -> Destination
* */
RowTransformBlock = new TransformBlock <string[], string[]>(inp => RowTransformFunction.Invoke(inp));
BatchTransformBlock = new TransformBlock <string[][], InMemoryTable>(inp => BatchTransformFunction.Invoke(inp));
DestinationBlock = new ActionBlock <InMemoryTable>(outp => Destination.WriteBatch(outp));
SourceBufferBlock.LinkTo(RowTransformBlock);
RowTransformBlock.LinkTo(DestinationBatchBlock);
DestinationBatchBlock.LinkTo(BatchTransformBlock);
BatchTransformBlock.LinkTo(DestinationBlock);
SourceBufferBlock.Completion.ContinueWith(t => { NLogger.Debug($"SoureBufferBlock DataFlow Completed: {TaskName}", TaskType, "RUN", TaskHash); RowTransformBlock.Complete(); });
RowTransformBlock.Completion.ContinueWith(t => { NLogger.Debug($"RowTransformBlock DataFlow Completed: {TaskName}", TaskType, "RUN", TaskHash); DestinationBatchBlock.Complete(); });
DestinationBatchBlock.Completion.ContinueWith(t => { NLogger.Debug($"DestinationBatchBlock DataFlow Completed: {TaskName}", TaskType, "RUN", TaskHash); BatchTransformBlock.Complete(); });
BatchTransformBlock.Completion.ContinueWith(t => { NLogger.Debug($"BatchTransformBlock DataFlow Completed: {TaskName}", TaskType, "RUN", TaskHash); DestinationBlock.Complete(); });
Source.Read(RowTransformBlock);
SourceBufferBlock.Complete();
DestinationBlock.Completion.Wait();
NLogger.Info(TaskName, TaskType, "END", TaskHash, ControlFlow.STAGE, ControlFlow.CurrentLoadProcess?.LoadProcessKey);
}
}
public async Task AnalyseAsync(string authorization, string inputSource, string reportOutput, bool doContentCheck = false)
{
if (string.IsNullOrEmpty(authorization))
{
throw new ArgumentNullException("You must provide the target bot (node) for this action.");
}
if (string.IsNullOrEmpty(inputSource))
{
throw new ArgumentNullException("You must provide the input source (phrase or file) for this action.");
}
if (string.IsNullOrEmpty(reportOutput))
{
throw new ArgumentNullException("You must provide the full output's report file name for this action.");
}
_logger.LogDebug("COMEÇOU!");
_fileService.CreateDirectoryIfNotExists(reportOutput);
var bucketStorage = new BucketStorage("Key " + authorization);
var contentProvider = new Take.ContentProvider.ContentProvider(bucketStorage, 5);
var client = _blipClientFactory.GetInstanceForAI(authorization);
var allIntents = new List <Intention>();
if (doContentCheck)
{
_logger.LogDebug("\tCarregando intencoes...");
allIntents = await client.GetAllIntentsAsync();
_logger.LogDebug("\tCarregadas!");
}
bool isPhrase = false;
var isDirectory = _fileService.IsDirectory(inputSource);
var isFile = _fileService.IsFile(inputSource);
if (isFile)
{
_logger.LogDebug("\tA entrada é um arquivo");
isPhrase = false;
}
else
if (isDirectory)
{
_logger.LogError("\tA entrada é um diretório");
throw new ArgumentNullException("You must provide the input source (phrase or file) for this action. Your input was a direcory.");
}
else
{
_logger.LogDebug("\tA entrada é uma frase");
isPhrase = true;
}
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = DataflowBlockOptions.Unbounded,
MaxDegreeOfParallelism = 20,
};
var analyseBlock = new TransformBlock <NLPAnalyseDataBlock, NLPAnalyseDataBlock>((Func <NLPAnalyseDataBlock, Task <NLPAnalyseDataBlock> >)AnalyseForMetrics, options);
var checkBlock = new TransformBlock <NLPAnalyseDataBlock, NLPAnalyseDataBlock>((Func <NLPAnalyseDataBlock, NLPAnalyseDataBlock>)CheckResponse, options);
var contentBlock = new TransformBlock <NLPAnalyseDataBlock, NLPAnalyseDataBlock>((Func <NLPAnalyseDataBlock, Task <NLPAnalyseDataBlock> >)GetContent, options);
var showResultBlock = new ActionBlock <NLPAnalyseDataBlock>(BuildResult, new ExecutionDataflowBlockOptions
{
BoundedCapacity = DataflowBlockOptions.Unbounded,
MaxMessagesPerTask = 1
});
analyseBlock.LinkTo(checkBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
checkBlock.LinkTo(contentBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
contentBlock.LinkTo(showResultBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
_count = 0;
var inputList = await GetInputList(isPhrase, inputSource, client, reportOutput, allIntents, contentProvider, doContentCheck);
_total = inputList.Count;
foreach (var input in inputList)
{
await analyseBlock.SendAsync(input);
}
analyseBlock.Complete();
await showResultBlock.Completion;
_logger.LogDebug("TERMINOU!");
}
public void ProduceLogs(int count, int buffSize)
{
var writerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 10 };
var serializerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = buffSize, MaxDegreeOfParallelism = 8, SingleProducerConstrained = true };
LogGenerator g = new LogGenerator();
var file = new StreamWriter("basic.async.srlz.log", false);
TransformBlock<LogEntry, string> serializer = new TransformBlock<LogEntry, string>(
e => string.Format(e.format, e.parameters),
serializerOptions);
ActionBlock<string> writer = new ActionBlock<string>(s => file.WriteLine(s), writerOptions);
serializer.LinkTo(writer, new DataflowLinkOptions() { PropagateCompletion = true });
for (int i = 0; i < count; i++)
{
g.Next();
var entry = new LogEntry() { format = g.FormatStr, parameters = new object[] { g.Param1, g.Param2, g.Param3, g.Param4, g.Param5, g.Param6 } };
serializer.SendAsync(entry).Wait();
}
serializer.Complete();
Completed = writer.Completion.ContinueWith(t => file.Close());
}
//[Fact(Skip = "Outerloop")]
public void RunTransformBlockConformanceTests()
{
bool passed = true;
// SYNC
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformBlockFactory =
options =>
{
TransformBlock<int, int> transformBlock = new TransformBlock<int, int>(i => i, (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, dopOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, cancellationOptions, 1, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return i;
});
for (int i = 0; i < 2; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var t = new TransformBlock<int, int>(i => i, dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformBlock<int, int>(new Func<int, int>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
// ASYNC (a copy of the sync but with constructors returning Task<T> instead of T
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformBlockFactory =
options =>
{
TransformBlock<int, int> transformBlock = new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i), (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, dopOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return Task.Factory.StartNew(() => i);
});
for (int i = 0; i < 2; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if ((i % 2) == 0) return null;
return Task.Factory.StartNew(() => i);
});
for (int i = 0; i < 2; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: null tasks are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if (i == 0)
{
Task.Delay(10).Wait();
return null;
}
return Task.Factory.StartNew(() => i);
}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 });
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformBlock<int, int>(new Func<int, Task<int>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformBlock<int, int>(new Func<int, Task<int>>(i => Task<int>.Factory.StartNew(() => { throw new InvalidOperationException(); })));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
public static async Task Compress(Stream inputStream, Stream outputStream)
{
var buffer = new BufferBlock<CompressionDetails>(new DataflowBlockOptions { BoundedCapacity = BoundedCapacity });
var compressorOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelism,
BoundedCapacity = BoundedCapacity
};
var writerOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = BoundedCapacity,
SingleProducerConstrained = true
};
var compressor = new TransformBlock<CompressionDetails, CompressionDetails>(compressionDetails => Compress(compressionDetails), compressorOptions);
var writer = new ActionBlock<CompressionDetails>(compressionDetailsWithSize => Multiplex(outputStream, compressionDetailsWithSize), writerOptions);
buffer.LinkTo(compressor);
compressor.LinkTo(writer);
buffer.Completion.ContinueWith(task => compressor.Complete());
compressor.Completion.ContinueWith(task => writer.Complete());
long sourceLength = inputStream.Length;
// Write total size to destination
byte[] size = BitConverter.GetBytes(sourceLength);
await outputStream.WriteAsync(size, 0, size.Length);
long chunkSize = 1048576; // 1 MB
int index = 0;
while (sourceLength > 0)
{
byte[] data = new byte[chunkSize];
int readCount = await inputStream.ReadAsync(data, 0, data.Length);
byte[] bytes = new byte[readCount];
Buffer.BlockCopy(data, 0, bytes, 0, readCount);
CompressionDetails compressionDetails = new CompressionDetails
{
Bytes = bytes,
ChunkSize = BitConverter.GetBytes(chunkSize),
Sequence = ++index
};
while (await buffer.SendAsync(compressionDetails) != true) { }
sourceLength -= chunkSize;
if (sourceLength < chunkSize)
chunkSize = sourceLength;
if (sourceLength == 0)
buffer.Complete();
}
writer.Completion.Wait();
await outputStream.FlushAsync();
inputStream.Dispose();
outputStream.Dispose();
}
public async Task TestPrecanceled()
{
var bb = new TransformBlock<int, int>(i => i,
new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) });
int ignoredValue;
IList<int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget<int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
public void Complete()
{
transformBlock.Complete();
}
public async Task TestNullTasksIgnored()
{
foreach (int dop in new[] { DataflowBlockOptions.Unbounded, 1, 2 })
{
var tb = new TransformBlock<int, int>(i => {
if ((i % 2) == 0) return null;
return Task.Run(() => i);
}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop });
const int Iters = 100;
tb.PostRange(0, Iters);
tb.Complete();
for (int i = 0; i < Iters; i++)
{
if ((i % 2) != 0)
{
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
}
}
await tb.Completion;
}
}
///<summary>全ハッシュを一定サイズに分割してソートする
///分割されたファイルをマージソートすると完全なソート済み列が得られる</summary>
public static async Task <int> QuickSortAll(int Index, long SortMask)
{
int FileCount = 0;
var SortComp = new BlockSortComparer(SortMask);
//これにソート用の配列を入れてメモリ割り当てを減らしてみる
var LongPool = new ConcurrentBag <long[]>();
bool LongPoolReturn = true;
//ソート用の配列の個数に制限を設ける
var FirstSortSemaphore = new SemaphoreSlim(config.hash.InitialSortConcurrency);
//ソートは並列 書き込みは並列させない
var FirstSortBlock = new TransformBlock <FirstSort, FirstSort>(async(t) =>
{
await QuickSortParllel(SortMask, t.ToSort, t.Length, SortComp).ConfigureAwait(false);
return(t);
}, new ExecutionDataflowBlockOptions()
{
SingleProducerConstrained = true,
MaxDegreeOfParallelism = config.hash.InitialSortConcurrency,
});
var WriterBlock = new ActionBlock <FirstSort>((t) =>
{
using (var writer = new UnbufferedLongWriter(t.WriteFilePath))
{
writer.WriteDestructive(t.ToSort, t.Length);
}
//ここでLongPoolに配列を返却する
if (LongPoolReturn)
{
LongPool.Add(t.ToSort);
}
FirstSortSemaphore.Release();
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
FirstSortBlock.LinkTo(WriterBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
//まずはAllHashを読む
using (var reader = new UnbufferedLongReader(AllHashFilePath))
{
for (; reader.Readable; FileCount++)
{
await FirstSortSemaphore.WaitAsync().ConfigureAwait(false);
if (!LongPool.TryTake(out var ToSort))
{
ToSort = new long[InitialSortUnit];
}
int ToSortLength = reader.Read(ToSort);
FirstSortBlock.Post(new FirstSort(SortingFilePath(Index, FileCount), ToSort, ToSortLength));
}
}
//NewerHashを読む
int ToSortNewerCursor = 0;
await FirstSortSemaphore.WaitAsync().ConfigureAwait(false);
if (!LongPool.TryTake(out var ToSortNewer))
{
ToSortNewer = new long[InitialSortUnit];
}
foreach (var filePath in Directory.EnumerateFiles(config.hash.TempDir, Path.GetFileName(NewerHashFilePathBase("*"))))
{
using (var reader = new BufferedLongReader(filePath))
{
while (reader.Readable)
{
for (; ToSortNewerCursor < ToSortNewer.Length; ToSortNewerCursor++)
{
if (!reader.MoveNext(out var next))
{
break;
}
ToSortNewer[ToSortNewerCursor] = next;
}
if (InitialSortUnit <= ToSortNewerCursor)
{
FirstSortBlock.Post(new FirstSort(SortingFilePath(Index, FileCount), ToSortNewer, ToSortNewer.Length));
FileCount++;
ToSortNewerCursor = 0;
await FirstSortSemaphore.WaitAsync().ConfigureAwait(false);
if (!LongPool.TryTake(out ToSortNewer))
{
ToSortNewer = new long[InitialSortUnit];
}
}
}
}
}
//余った要素もソートさせる FirstSortingCountはもう使わないので放置
if (0 < ToSortNewerCursor)
{
FirstSortBlock.Post(new FirstSort(SortingFilePath(Index, FileCount), ToSortNewer, ToSortNewerCursor));
FileCount++;
}
FirstSortBlock.Complete();
//ソート用配列は作り終わったので用が済んだ配列は解放させる
LongPoolReturn = false;
LongPool.Clear();
await WriterBlock.Completion.ConfigureAwait(false);
return(FileCount);
}
/// <summary>
/// Recieve Data
/// |
/// Broadcast
/// __________________|_________________
/// Document Involved Parties Order
/// |__________________|_________________|
/// |
/// Compile InformationS
/// </summary>
public void StartPipeline()
{
////TODO Need to add Buffer block to recieve batch post, then fill pipeline
//Broadcasts the received data
broadcastOrder = new BroadcastBlock <Order>(order => order);
//Joins all the parallel processes together to output for compiling
var joinblock = new JoinBlock <DocumentTransform, InvolvedPartyTransforms, OrderTransform>(new GroupingDataflowBlockOptions {
Greedy = false
});
//Add cancelation to the pipeline
cancellationTokenSource = new CancellationTokenSource();
ExecutionDataflowBlockOptions executionDataflowBlockOptions = new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationTokenSource.Token
};
//Receives input orders
receiveData = new TransformBlock <Order, Order>(order =>
{
return(order != null ? order : null);
}, executionDataflowBlockOptions);
//Processes Client/Lender information
processInvolvedParties = new TransformBlock <Order, InvolvedPartyTransforms>(order =>
{
if (ClientVerification.VerifyClient(order.clientInformation).Result&& ClientVerification.VerifyLender(order.lenderInformation).Result)
{
ClientTransform clientTransform = new ClientTransform
{
ClientInformation = order.clientInformation
};
LenderTransform lenderTransform = new LenderTransform
{
lenderInformation = order.lenderInformation
};
InvolvedPartyTransforms involvedPartyTransforms = new InvolvedPartyTransforms
{
lenderTransform = lenderTransform,
clientTransform = clientTransform
};
return(involvedPartyTransforms);
}
return(null);
}, executionDataflowBlockOptions);
//Processes Order Information
processOrder = new TransformBlock <Order, OrderTransform>(order =>
{
OrderTransform orderTransform = new OrderTransform
{
order = order
};
return(orderTransform);
}, executionDataflowBlockOptions);
//TODO add more processing
//Processes Documents from order
processDocuments = new TransformBlock <Order, DocumentTransform>(order =>
{
List <Document> documents = order.documents;
//if(documents.Count < 0)
//{ return null; }
//foreach(Document document in documents)
//{
// if(document == null)
// { return null; }
//}
//TODO generate events
DocumentTransform documentTransform = new DocumentTransform
{
documents = documents
};
return(documentTransform);
}, executionDataflowBlockOptions);
//Compiles the information back after processing
compileInformation = new TransformBlock <Tuple <DocumentTransform, InvolvedPartyTransforms, OrderTransform>, Order>((transforms) =>
{
Console.WriteLine("Compiling Order #{0}", transforms.Item3.order.orderId);
Console.WriteLine("Lender: {0} | Client: {1}",
transforms.Item2.lenderTransform.lenderInformation.lenderName,
transforms.Item2.clientTransform.ClientInformation.clientFName + ' ' +
transforms.Item2.clientTransform.ClientInformation.clientLName);
return(transforms.Item3.order);
}, executionDataflowBlockOptions);
//Linking options
var options = new DataflowLinkOptions {
PropagateCompletion = true
};
receiveData.LinkTo(broadcastOrder);
//Broadcasts received data to processing blocks
broadcastOrder.LinkTo(processInvolvedParties);
broadcastOrder.LinkTo(processOrder);
broadcastOrder.LinkTo(processDocuments);
processDocuments.LinkTo(joinblock.Target1, options);
processInvolvedParties.LinkTo(joinblock.Target2, options);
processOrder.LinkTo(joinblock.Target3, options);
joinblock.LinkTo(compileInformation, options);
//Creates random orders and posts them to the Pipeline Batchblock
//for (int i = 0; i < 50; i++)
//{
// Random random = new Random();
// Order order = new Order().RandomOrder();
//}
//Currently is just manually loading two orders
Order orderRand = new Order().RandomOrder();
var anotherOrder = new Order().RandomOrder();
receiveData.Post(orderRand);
receiveData.Post(anotherOrder);
receiveData.Complete();
const int TIMEOUT = 30000;
//compileInformation.Completion.Wait(TIMEOUT);
//Currently is just manually receiving two orders
var test = compileInformation.ReceiveAsync(cancellationTokenSource.Token);
Console.WriteLine("One: " + test.Result.orderId);
test = compileInformation.ReceiveAsync(cancellationTokenSource.Token);
Console.WriteLine("Two: " + test.Result.orderId);
}
public void AsyncNullTest ()
{
var scheduler = new TestScheduler ();
var block = new TransformBlock<int, int> (
i => null,
new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
block.Complete ();
Assert.IsTrue (block.Completion.Wait (100));
}
public static async Task <IEnumerable <Activity> > GetActivities(Folder folder, Route route, CancellationToken token)
{
if (null == folder)
{
throw new ArgumentNullException(nameof(folder));
}
if (null == route)
{
throw new ArgumentNullException(nameof(route));
}
using (SemaphoreSlim addItem = new SemaphoreSlim(1))
{
List <Activity> result = new List <Activity>();
string activitiesDirectory = route.RouteFolder.ActivitiesFolder;
result.Add(DefaultExploreActivity);
result.Add(ExploreThroughActivity);
if (Directory.Exists(activitiesDirectory))
{
TransformBlock <string, Activity> inputBlock = new TransformBlock <string, Activity>
(activityFile =>
{
return(FromPath(activityFile, folder, route));
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = System.Environment.ProcessorCount, CancellationToken = token
});
ActionBlock <Activity> actionBlock = new ActionBlock <Activity>
(async activity =>
{
if (activity == null)
{
return;
}
try
{
await addItem.WaitAsync(token).ConfigureAwait(false);
result.Add(activity);
}
finally
{
addItem.Release();
}
});
inputBlock.LinkTo(actionBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (string activityFile in Directory.EnumerateFiles(activitiesDirectory, "*.act"))
{
await inputBlock.SendAsync(activityFile).ConfigureAwait(false);
}
inputBlock.Complete();
await actionBlock.Completion.ConfigureAwait(false);
}
return(result);
}
}
/// <summary>
/// THIS CODE IS JUST EXAMPLE AND NOT INTENDED FOR PRODUCTION USE
/// </summary>
/// <param name="args"></param>
static void Main(string[] args)
{
// Create the cancellation source.
var cancellationSource = new CancellationTokenSource();
var inputWorkBufferBlock = new BufferBlock<Uri>();
// Input - Uri - seed address
// Output - Uri - key, content, content-type
var downloaderBlock = new TransformBlock<Uri, string>(address =>
{
var httpClient = new HttpClient();
// Downloads the requested resource as a string.
Console.WriteLine("Downloading '{0}'... Thread id {1}", address.OriginalString, Thread.CurrentThread.ManagedThreadId);
var contentType = string.Empty;
var content = httpClient.GetAsync(address).ContinueWith(task =>
{
HttpResponseMessage response = task.Result;
if (task.Result.IsSuccessStatusCode)
{
return task.Result.Content.ReadAsStringAsync();
}
return new Task<string>(() => null);
}).Unwrap();
return content.Result;
}, new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationSource.Token,
MaxDegreeOfParallelism = 5
});
var outputBufferBlock = new BufferBlock<string>();
var saverBlock = new ActionBlock<string>(content =>
{
if (content != null)
{
const string targetPath = "c:\\work\\tmp";
const string extension = ".html";
var fileName = Path.ChangeExtension(Path.Combine(targetPath, Path.GetRandomFileName()), extension);
Console.WriteLine("Saving {0} ...Thread: {1}", fileName, Thread.CurrentThread.ManagedThreadId);
using (var stream = new StreamWriter(fileName))
{
stream.Write(content);
}
}
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
// Blocks linking
inputWorkBufferBlock.LinkTo(downloaderBlock);
// Filtering, skips empty response
downloaderBlock.LinkTo(outputBufferBlock, s => !string.IsNullOrWhiteSpace(s));
outputBufferBlock.LinkTo(saverBlock);
// Propagating completition
inputWorkBufferBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)downloaderBlock).Fault(t.Exception);
else downloaderBlock.Complete();
});
downloaderBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)outputBufferBlock).Fault(t.Exception);
else outputBufferBlock.Complete();
});
outputBufferBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)outputBufferBlock).Fault(t.Exception);
else outputBufferBlock.Complete();
});
outputBufferBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)saverBlock).Fault(t.Exception);
else saverBlock.Complete();
});
// Message passing
inputWorkBufferBlock.Post(new Uri("http://svnbook.red-bean.com/nightly/ru/svn-book.html"));
inputWorkBufferBlock.Post(new Uri("http://bash.im"));
inputWorkBufferBlock.Post(new Uri("http://habrahabr.ru"));
inputWorkBufferBlock.Post(new Uri("http://lb.ua"));
inputWorkBufferBlock.Post(new Uri("http://blogs.msdn.com/b/pfxteam/"));
inputWorkBufferBlock.Post(new Uri("http://hgbook.red-bean.com/read/a-tour-of-mercurial-merging-work.html"));
inputWorkBufferBlock.Complete();
saverBlock.Completion.Wait();
Console.WriteLine("Job is DONE...");
Console.WriteLine("Hit ANY KEY to exit...");
Console.ReadKey();
}
public void Process(List <string> filters, CancellationToken token)
{
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
Func <IServiceMessage, RoutedMessage> partitioner = x => new RoutedMessage
{
Message = x,
PartitionId = x.Id.GetHashCode() / 1000000000
};
var partitionerBlock = new TransformBlock <IServiceMessage, RoutedMessage>(partitioner);
var actionBlock1 = new ActionBlock <RoutedMessage>(async(RoutedMessage msg) => await _service.ProcessAsync(msg));
var actionBlock2 = new ActionBlock <RoutedMessage>(async(RoutedMessage msg) => await _service.ProcessAsync(msg));
var actionBlock3 = new ActionBlock <RoutedMessage>(async(RoutedMessage msg) => await _service.ProcessAsync(msg));
var actionBlock4 = new ActionBlock <RoutedMessage>(async(RoutedMessage msg) => await _service.ProcessAsync(msg));
var actionBlock5 = new ActionBlock <RoutedMessage>(async(RoutedMessage msg) => await _service.ProcessAsync(msg));
var actionBlock6 = new ActionBlock <RoutedMessage>(async(RoutedMessage msg) => await _service.ProcessAsync(msg));
var actionBlock7 = new ActionBlock <RoutedMessage>(async(RoutedMessage msg) => await _service.ProcessAsync(msg));
var actionBlock8 = new ActionBlock <RoutedMessage>(async(RoutedMessage msg) => await _service.ProcessAsync(msg));
var actionBlock9 = new ActionBlock <RoutedMessage>(async(RoutedMessage msg) => await _service.ProcessAsync(msg));
partitionerBlock.LinkTo(actionBlock1, linkOptions, msg => msg.PartitionId == -4);
partitionerBlock.LinkTo(actionBlock1, linkOptions, msg => msg.PartitionId == -3);
partitionerBlock.LinkTo(actionBlock1, linkOptions, msg => msg.PartitionId == -2);
partitionerBlock.LinkTo(actionBlock1, linkOptions, msg => msg.PartitionId == -1);
partitionerBlock.LinkTo(actionBlock1, linkOptions, msg => msg.PartitionId == 0);
partitionerBlock.LinkTo(actionBlock1, linkOptions, msg => msg.PartitionId == 1);
partitionerBlock.LinkTo(actionBlock1, linkOptions, msg => msg.PartitionId == 2);
partitionerBlock.LinkTo(actionBlock1, linkOptions, msg => msg.PartitionId == 3);
partitionerBlock.LinkTo(actionBlock1, linkOptions, msg => msg.PartitionId == 4);
var tasks = new List <Task>();
foreach (var filter in filters)
{
tasks.Add(Task.Run(async() =>
{
Guid filterId = Guid.NewGuid();
while (!token.IsCancellationRequested)
{
var message = await _serviceClient.GetAsync(filter);
message.Id = filterId;
await partitionerBlock.SendAsync(message);
}
}));
}
while (!token.IsCancellationRequested)
{
Thread.Sleep(100);
}
partitionerBlock.Complete();
actionBlock1.Completion.Wait();
actionBlock2.Completion.Wait();
actionBlock3.Completion.Wait();
actionBlock4.Completion.Wait();
actionBlock5.Completion.Wait();
actionBlock6.Completion.Wait();
actionBlock7.Completion.Wait();
actionBlock8.Completion.Wait();
actionBlock9.Completion.Wait();
Task.WaitAll(tasks.ToArray(), 10000);
}
public Task RunAsync()
{
TransformBlock<Uri, PropertyBag> ingestBlock = new TransformBlock<Uri, PropertyBag>(input =>
{
PropertyBag result = new PropertyBag
{
OriginalUrl = input.ToString(),
UserAgent = _userAgent,
Step = new CrawlStep(input, 0)
};
return result;
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelism
});
TransformBlock<PropertyBag, PropertyBag> ingestBlockForAggregation =
new TransformBlock<PropertyBag, PropertyBag>(input => input, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelism
});
CrawlIngestionHelper crawlIngestionHelper = new CrawlIngestionHelper(ingestBlockForAggregation, _userAgent);
TransformBlock<PropertyBag, PropertyBag>[] pipeline = Pipeline
.Select(pipelineStep =>
{
return new TransformBlock<PropertyBag, PropertyBag>(async propertyBag =>
{
if (propertyBag.StopPipelining)
{
return propertyBag;
}
try
{
propertyBag.StopPipelining = !await pipelineStep.Process(crawlIngestionHelper, propertyBag);
}
catch (Exception exception)
{
propertyBag.Exceptions.Add(exception);
}
return propertyBag;
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = pipelineStep.MaxDegreeOfParallelism
});
})
.ToArray();
ActionBlock<PropertyBag> terminationCheckerBlock = new ActionBlock<PropertyBag>(propertyBag =>
{
if (ingestBlock.InputCount == 0
&& ingestBlock.OutputCount == 0
&& !ingestBlock.Completion.IsCompleted
&& !ingestBlock.Completion.IsCanceled
&& !ingestBlock.Completion.IsFaulted
&& ingestBlockForAggregation.InputCount == 0
&& ingestBlockForAggregation.OutputCount == 0)
{
if (pipeline.Any(transformBlock => transformBlock.InputCount != 0 || transformBlock.OutputCount != 0))
{
return;
}
ingestBlock.Complete();
}
}, new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = 1});
ingestBlock.LinkTo(ingestBlockForAggregation, new DataflowLinkOptions {PropagateCompletion = true});
TransformBlock<PropertyBag, PropertyBag> previous = ingestBlockForAggregation;
foreach (TransformBlock<PropertyBag, PropertyBag> transformBlock in pipeline)
{
previous.LinkTo(transformBlock, new DataflowLinkOptions {PropagateCompletion = true});
previous = transformBlock;
}
previous.LinkTo(terminationCheckerBlock, new DataflowLinkOptions {PropagateCompletion = true});
foreach (Uri startUri in StartUris)
{
ingestBlock.Post(startUri);
}
return terminationCheckerBlock.Completion;
}
public static void Run()
{
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
TransformBlock <string, string> downloadString = new TransformBlock <string, string>(async uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
Console.WriteLine("ThreadId: " + Thread.CurrentThread.ManagedThreadId);
Console.WriteLine();
return(await new HttpClient().GetStringAsync(uri));
});
// Separates the specified text into an array of words.
TransformBlock <string, string[]> createWordList = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
Console.WriteLine("ThreadId: " + Thread.CurrentThread.ManagedThreadId);
Console.WriteLine();
// Remove common punctuation by replacing all non-letter characters
// with a space character.
char[] tokens = text.Select(c => char.IsLetter(c) ? c : ' ').ToArray();
text = new string(tokens);
// Separate the text into an array of words.
return(text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries));
});
// Removes short words and duplicates.
TransformBlock <string[], string[]> filterWordList = new TransformBlock <string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
Console.WriteLine("ThreadId: " + Thread.CurrentThread.ManagedThreadId);
Console.WriteLine();
return(words
.Where(word => word.Length > 3)
.Distinct()
.ToArray());
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
TransformManyBlock <string[], string> findReversedWords = new TransformManyBlock <string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
Console.WriteLine("ThreadId: " + Thread.CurrentThread.ManagedThreadId);
Console.WriteLine();
HashSet <string> wordsSet = new HashSet <string>(words);
return(from word in words.AsParallel()
let reverse = new string(word.Reverse().ToArray())
where word != reverse && wordsSet.Contains(reverse)
select word);
});
// Prints the provided reversed words to the console.
ActionBlock <string> printReversedWords = new ActionBlock <string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}", reversedWord, new string(reversedWord.Reverse().ToArray()));
Console.WriteLine("ThreadId: " + Thread.CurrentThread.ManagedThreadId);
Console.WriteLine();
});
//
// Connect the dataflow blocks to form a pipeline.
//
DataflowLinkOptions linkOptions = new DataflowLinkOptions
{
// Gets or sets whether the linked target will have completion and
// faulting notification propagated to it automatically
PropagateCompletion = true
};
downloadString.LinkTo(createWordList, linkOptions);
createWordList.LinkTo(filterWordList, linkOptions);
filterWordList.LinkTo(findReversedWords, linkOptions);
findReversedWords.LinkTo(printReversedWords, linkOptions);
// Process "The Iliad of Homer" by Homer.
/*This example uses DataflowBlock.Post to synchronously send data to the head
* of the pipeline. Use the DataflowBlock.SendAsync method when you must asynchronously
* send data to a dataflow node.*/
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
/*Add the following code to mark the head of the pipeline as completed.
* The head of the pipeline propagates its completion after it processes
* all buffered messages.*/
/*Signals to the IDataflowBlock that it should not accept nor produce any more
* messages nor consume any more postponed messages*/
downloadString.Complete();
/*Add the following code to wait for the pipeline to finish. The overall operation is
* finished when the tail of the pipeline finishes.*/
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
}
public async Task TestCancellationExceptionsIgnored()
{
var t = new TransformBlock<int, int>(i => {
if ((i % 2) == 0) throw new OperationCanceledException();
return i;
});
t.PostRange(0, 2);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0)
{
Assert.Equal(expected: i, actual: await t.ReceiveAsync());
}
}
await t.Completion;
}
public async Task <RemoteSegmentWithData[]> GetRemoteMetadata(ImportedEvent[] events)
{
// asynchronously start downloading all the metadata we need
Log.Info($"Begin downloading metadata for segments (Request concurrency: {this.maxDegreeOfParallelism})");
var groupedEvents = new ConcurrentDictionary <long, ConcurrentBag <ImportedEvent> >(
this.maxDegreeOfParallelism,
events.Length / 10);
// execution options allows us to throttle requests
var options = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = this.maxDegreeOfParallelism,
SingleProducerConstrained = true,
};
// the transform block maps A->B, ensuring that all events have an audio recording id
var getRecordingIdBlock = new TransformBlock <ImportedEvent, ImportedEvent>(
(importedEvent) => this.GetAudioRecordingId(importedEvent),
options);
// all events are buffered into groups based on audio recording id
var groupRecordingsBlock = new ActionBlock <ImportedEvent>(
importedEvent =>
{
var collection = groupedEvents.GetOrAdd(
importedEvent.AudioRecordingId.Value,
new ConcurrentBag <ImportedEvent>());
collection.Add(importedEvent);
});
// the metadata for each recording is retrieved and used to produce many segments (one per imported event)
var createSegmentsBlock = new TransformManyBlock <KeyValuePair <long, ConcurrentBag <ImportedEvent> >, RemoteSegmentWithData>(
(group) => this.DownloadRemoteMetadata(group.Key, group.Value),
options);
// the transform block can't `Complete` unless it's output is empty
// so add a buffer block to store the transform block's output
var bufferBlock = new BufferBlock <RemoteSegmentWithData>();
// link the two parts of block A
getRecordingIdBlock.LinkTo(groupRecordingsBlock);
// link the two parts of block B
createSegmentsBlock.LinkTo(bufferBlock);
// kick off the chain, resolve audio recording ids and group
foreach (var record in events)
{
// post an event to the transform block to process
getRecordingIdBlock.Post(record);
}
Log.Trace("Finished posting messages to recording id resolver");
getRecordingIdBlock.Complete();
Log.Trace("Waiting for getRecordingIdBlock to resolve");
await getRecordingIdBlock.Completion;
Log.Trace("Waiting for groupRecordingsBlock to resolve");
groupRecordingsBlock.Complete();
await groupRecordingsBlock.Completion;
var eventCount = groupedEvents.Sum(kvp => kvp.Value.Count);
Log.Trace($"Finished waiting for recording ids to resolve, {eventCount} events grouped into {groupedEvents.Count} recordings");
// now post the grouped audio recordings to the segment generating block
foreach (var keyValuePair in groupedEvents)
{
createSegmentsBlock.Post(keyValuePair);
}
Log.Trace("Finished posting messages to recording metadata downloader");
createSegmentsBlock.Complete();
// wait for all requests to finish
Log.Trace("Begin waiting for metadata downloader");
await createSegmentsBlock.Completion;
Log.Trace("Finished waiting for metadata downloader");
if (bufferBlock.TryReceiveAll(out var segments))
{
RemoteSegmentWithData[] segmentsArray;
int finalEventCount;
lock (segments)
{
segmentsArray = segments.ToArray();
// do some excessive logic checking because we used to have race conditions
finalEventCount = segmentsArray.Sum(x => x.Data.Count);
if (events.Length != finalEventCount)
{
throw new InvalidOperationException(
$"The number of supplied events ({events.Length}) did" +
$" not match the number of events that had metadata resolved ({finalEventCount})" +
" - a race condition has occurred");
}
}
Log.Info($"Metadata generated for {finalEventCount} events, {segmentsArray.Length} segments created");
return(segmentsArray);
}
else
{
throw new InvalidOperationException("Failed to retrieve media info from data flow.");
}
}
[InlineData(2, 1, false)] // no force ordered, but dop == 1, so it doesn't matter
public async Task TestOrdering_Async_OrderedEnabled(int mmpt, int dop, bool? EnsureOrdered)
{
const int iters = 1000;
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, MaxMessagesPerTask = mmpt };
if (EnsureOrdered == null)
{
Assert.True(options.EnsureOrdered);
}
else
{
options.EnsureOrdered = EnsureOrdered.Value;
}
var tb = new TransformBlock<int, int>(i => Task.FromResult(i), options);
tb.PostRange(0, iters);
for (int i = 0; i < iters; i++)
{
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
}
tb.Complete();
await tb.Completion;
}
public static void main()
{
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock <string, string>(async uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return(await new HttpClient().GetStringAsync(uri));
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character.
char[] tokens = text.Select(c => char.IsLetter(c) ? c : ' ').ToArray();
text = new string(tokens);
// Separate the text into an array of words.
return(text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries));
});
// Removes short words and duplicates.
var filterWordList = new TransformBlock <string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return(words
.Where(word => word.Length > 3)
.Distinct()
.ToArray());
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock <string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
var wordsSet = new HashSet <string>(words);
return(from word in words.AsParallel()
let reverse = new string(word.Reverse().ToArray())
where word != reverse && wordsSet.Contains(reverse)
select word);
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock <string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
//
// Connect the dataflow blocks to form a pipeline.
//
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
downloadString.LinkTo(createWordList, linkOptions);
createWordList.LinkTo(filterWordList, linkOptions);
filterWordList.LinkTo(findReversedWords, linkOptions);
findReversedWords.LinkTo(printReversedWords, linkOptions);
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/cache/epub/16452/pg16452.txt");
// Mark the head of the pipeline as complete.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
}
public async Task TestOrdering_Sync_OrderedDisabled()
{
// If ordering were enabled, this test would hang.
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, EnsureOrdered = false };
var mres = new ManualResetEventSlim();
var tb = new TransformBlock<int, int>(i =>
{
if (i == 0) mres.Wait();
return i;
}, options);
tb.Post(0);
tb.Post(1);
Assert.Equal(1, await tb.ReceiveAsync());
mres.Set();
Assert.Equal(0, await tb.ReceiveAsync());
tb.Complete();
await tb.Completion;
}
internal static bool TenTransformsToAction()
{
const int ITERS = 2;
var first = new TransformBlock<int, int>(item => item);
TransformBlock<int, int> t = first;
for (int i = 0; i < 9; i++)
{
var next = new TransformBlock<int, int>(item => item);
t.LinkWithCompletion(next);
t = next;
}
int completedCount = 0;
var last = new ActionBlock<int>(i => completedCount++);
t.LinkWithCompletion(last);
for (int i = 0; i < ITERS; i++) first.Post(i);
first.Complete();
last.Completion.Wait();
return completedCount == ITERS;
}
