public void TestSwitch() { var input = new[] { 1, 2, 3, 4, 5 }; Predicate <int> predicate = i => i % 2 == 0; var target1 = new BufferBlock <int>(); var target2 = new BufferBlock <int>(); var inputBlock = new TransformBlock <int, int>(i => i); inputBlock .LinkWhen(predicate, target1) .LinkOtherwise(target2); input.ToObservable() .Subscribe(inputBlock.AsObserver()); IList <int> output1 = target1 .AsObservable() .ToEnumerable() .ToList(); IList <int> output2 = target2 .AsObservable() .ToEnumerable() .ToList(); output1.Should() .BeEquivalentTo(input.Where(i => predicate(i))); output2.Should() .BeEquivalentTo(input.Where(i => !predicate(i))); }
public static void Transform(string fileIn, string fileOut, Func <JsonTextReader, JsonTextWriter, bool> transform) { using (var reader = new StreamReader(fileIn, Encoding.UTF8)) using (var writer = new StreamWriter(fileOut, false, Encoding.UTF8)) { var transformBlock = new TransformBlock <string, string>( evt => { var stringWriter = new StringWriter(); using (var jsonReader = new JsonTextReader(new StringReader(evt))) using (var jsonWriter = new JsonTextWriter(stringWriter)) { while (jsonReader.Read()) { if (!transform(jsonReader, jsonWriter)) { jsonWriter.WriteToken(jsonReader.TokenType, jsonReader.Value); } } } return(stringWriter.ToString()); }, new ExecutionDataflowBlockOptions { BoundedCapacity = 1024, MaxDegreeOfParallelism = 8 // TODO:parameterize }); var outputBock = new ActionBlock <string>(l => { if (!string.IsNullOrEmpty(l)) { writer.WriteLine(l); } }, new ExecutionDataflowBlockOptions { BoundedCapacity = 1024, MaxDegreeOfParallelism = 1 }); transformBlock.LinkTo(outputBock, new DataflowLinkOptions { PropagateCompletion = true }); var input = transformBlock.AsObserver(); string line; while ((line = reader.ReadLine()) != null) { input.OnNext(line); } input.OnCompleted(); outputBock.Completion.Wait(); } }
public static ISourceBlock <List <T> > CreateBatch <T>(string file, Func <Line, T> transform) { var block = new TransformBlock <List <Line>, List <T> >( batch => batch.Select(transform).ToList(), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8, BoundedCapacity = 128 }); new Thread(() => { var input = block.AsObserver(); try { using (var reader = new StreamReader(file)) { string line; int lineNr = 0; var batch = new List <Line>(); while ((line = reader.ReadLine()) != null) { batch.Add(new Line { Content = line, Number = lineNr++ }); if (batch.Count >= 512) { input.OnNext(batch); batch = new List <Line>(); } } if (batch.Count > 0) { input.OnNext(batch); } input.OnCompleted(); } } catch (Exception ex) { input.OnError(ex); } }).Start(); return(block); // var order = new TransformBlock<List<T>, List<T>>(t => t, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }); // block.LinkTo(order, new DataflowLinkOptions { PropagateCompletion = true }); // return order; }
public static ISourceBlock <T> Create <T>(string file, Func <Line, T> transform) { var block = new TransformBlock <Line, T>( transform, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, BoundedCapacity = 128 }); new Thread(() => { var input = block.AsObserver(); try { using (var reader = new StreamReader(file)) { string line; int lineNr = 0; var batch = new List <Line>(); while ((line = reader.ReadLine()) != null) { input.OnNext(new Line { Content = line, Number = lineNr++ }); } input.OnCompleted(); } } catch (Exception ex) { input.OnError(ex); } }).Start(); return(block); }
public static void Convert(StreamReader reader, StreamWriter writer) { var line = reader.ReadLine(); if (line == null) { return; } var jExample = JObject.Parse(line); var settings = jExample.Properties().Any(p => p.Name == "_multi") ? "--cb_explore_adf" : "--cb_explore"; int lineNr = 1; using (var vw = new VowpalWabbit(new VowpalWabbitSettings(settings) { EnableStringExampleGeneration = true, EnableStringFloatCompact = true, EnableThreadSafeExamplePooling = true })) { var serializeBlock = new TransformBlock <Tuple <string, int>, string>(l => { using (var jsonSerializer = new VowpalWabbitJsonSerializer(vw)) using (var example = jsonSerializer.ParseAndCreate(l.Item1)) { if (example == null) { throw new InvalidDataException($"Invalid example in line {l.Item2}: '{l.Item1}'"); } var str = example.VowpalWabbitString; if (example is VowpalWabbitMultiLineExampleCollection) { str += "\n"; } return(str); } }, new ExecutionDataflowBlockOptions { BoundedCapacity = 1024, MaxDegreeOfParallelism = 8 }); var writeBlock = new ActionBlock <string>( l => writer.WriteLine(l), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, BoundedCapacity = 128 }); serializeBlock.LinkTo(writeBlock, new DataflowLinkOptions { PropagateCompletion = true }); var input = serializeBlock.AsObserver(); do { input.OnNext(Tuple.Create(line, lineNr)); lineNr++; } while ((line = reader.ReadLine()) != null); input.OnCompleted(); serializeBlock.Completion.Wait(); } }
static void Main(string[] args) { // // Create the members of the pipeline. // // Downloads the requested resource as a string. var downloadString = new TransformBlock<string, string>(uri => { Console.WriteLine("Downloading '{0}'...", uri); return new WebClient().DownloadString(uri); }); // Separates the specified text into an array of words. var createWordList = new TransformBlock<string, string[]>(text => { Console.WriteLine("Creating word list..."); // Remove common punctuation by replacing all non-letter characters // with a space character to. char[] tokens = text.ToArray(); for (int i = 0; i < tokens.Length; i++) { if (!char.IsLetter(tokens[i])) tokens[i] = ' '; } text = new string(tokens); // Separate the text into an array of words. return text.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); }); // Removes short words, orders the resulting words alphabetically, // and then remove duplicates. var filterWordList = new TransformBlock<string[], string[]>(words => { Console.WriteLine("Filtering word list..."); return words.Where(word => word.Length > 3).OrderBy(word => word) .Distinct().ToArray(); }); // Finds all words in the specified collection whose reverse also // exists in the collection. var findReversedWords = new TransformManyBlock<string[], string>(words => { Console.WriteLine("Finding reversed words..."); // Holds reversed words. var reversedWords = new ConcurrentQueue<string>(); // Add each word in the original collection to the result whose // reversed word also exists in the collection. Parallel.ForEach(words, word => { // Reverse the work. string reverse = new string(word.Reverse().ToArray()); // Enqueue the word if the reversed version also exists // in the collection. if (Array.BinarySearch<string>(words, reverse) >= 0 && word != reverse) { reversedWords.Enqueue(word); } }); return reversedWords; }); // Prints the provided reversed words to the console. var printReversedWords = new ActionBlock<string>(reversedWord => { Console.WriteLine("Found reversed words {0}/{1}", reversedWord, new string(reversedWord.Reverse().ToArray())); }); // // Connect the dataflow blocks to form a pipeline. // downloadString.AsObservable().Subscribe(i => Console.WriteLine("Action Sub {0}", i)); var te= downloadString.AsObserver(); downloadString.LinkTo(createWordList); createWordList.LinkTo(filterWordList); filterWordList.LinkTo(findReversedWords); findReversedWords.LinkTo(printReversedWords); // // For each completion task in the pipeline, create a continuation task // that marks the next block in the pipeline as completed. // A completed dataflow block processes any buffered elements, but does // not accept new elements. // downloadString.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)createWordList).Fault(t.Exception); else createWordList.Complete(); }); createWordList.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)filterWordList).Fault(t.Exception); else filterWordList.Complete(); }); filterWordList.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)findReversedWords).Fault(t.Exception); else findReversedWords.Complete(); }); findReversedWords.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)printReversedWords).Fault(t.Exception); else printReversedWords.Complete(); }); // Process "The Iliad of Homer" by Homer. downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt"); // Mark the head of the pipeline as complete. The continuation tasks // propagate completion through the pipeline as each part of the // pipeline finishes. downloadString.Complete(); // Wait for the last block in the pipeline to process all messages. printReversedWords.Completion.Wait(); Console.ReadLine(); te.OnNext("ciao"); Console.ReadLine(); }
static void Main(string[] args) { // // Create the members of the pipeline. // //var nums = Enumerable.Range(0, 10).ToList();//.AsQueryable(); //var query = (from num in nums.AsQueryable() // where num % 2 == 0 // select num * num).Sum(); //int sum = 0; //for (int index = 0; index < nums.Count; index++) //{ // int num = nums[index]; // if (num % 2 == 0) // sum += num * num; //} // Downloads the requested resource as a string. var downloadString = new TransformBlock <string, string>(uri => { Console.WriteLine("Downloading '{0}'...", uri); return(new WebClient().DownloadString(uri)); }); // Separates the specified text into an array of words. var createWordList = new TransformBlock <string, string[]>(text => { Console.WriteLine("Creating word list..."); // Remove common punctuation by replacing all non-letter characters // with a space character to. char[] tokens = text.ToArray(); for (int i = 0; i < tokens.Length; i++) { if (!char.IsLetter(tokens[i])) { tokens[i] = ' '; } } text = new string(tokens); // Separate the text into an array of words. return(text.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)); }); // Removes short words, orders the resulting words alphabetically, // and then remove duplicates. var filterWordList = new TransformBlock <string[], string[]>(words => { Console.WriteLine("Filtering word list..."); return(words.Where(word => word.Length > 3).OrderBy(word => word) .Distinct().ToArray()); }); // Finds all words in the specified collection whose reverse also // exists in the collection. var findReversedWords = new TransformManyBlock <string[], string>(words => { Console.WriteLine("Finding reversed words..."); // Holds reversed words. var reversedWords = new ConcurrentQueue <string>(); // Add each word in the original collection to the result whose // reversed word also exists in the collection. Parallel.ForEach(words, word => { // Reverse the work. string reverse = new string(word.Reverse().ToArray()); // Enqueue the word if the reversed version also exists // in the collection. if (Array.BinarySearch <string>(words, reverse) >= 0 && word != reverse) { reversedWords.Enqueue(word); } }); return(reversedWords); }); // Prints the provided reversed words to the console. var printReversedWords = new ActionBlock <string>(reversedWord => { Console.WriteLine("Found reversed words {0}/{1}", reversedWord, new string(reversedWord.Reverse().ToArray())); }); // // Connect the dataflow blocks to form a pipeline. // downloadString.AsObservable().Subscribe(i => Console.WriteLine("Action Sub {0}", i)); var te = downloadString.AsObserver(); downloadString.LinkTo(createWordList); createWordList.LinkTo(filterWordList); filterWordList.LinkTo(findReversedWords); findReversedWords.LinkTo(printReversedWords); // // For each completion task in the pipeline, create a continuation task // that marks the next block in the pipeline as completed. // A completed dataflow block processes any buffered elements, but does // not accept new elements. // downloadString.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)createWordList).Fault(t.Exception); } else { createWordList.Complete(); } }); createWordList.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)filterWordList).Fault(t.Exception); } else { filterWordList.Complete(); } }); filterWordList.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)findReversedWords).Fault(t.Exception); } else { findReversedWords.Complete(); } }); findReversedWords.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)printReversedWords).Fault(t.Exception); } else { printReversedWords.Complete(); } }); // Process "The Iliad of Homer" by Homer. downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt"); // Mark the head of the pipeline as complete. The continuation tasks // propagate completion through the pipeline as each part of the // pipeline finishes. downloadString.Complete(); // Wait for the last block in the pipeline to process all messages. printReversedWords.Completion.Wait(); Console.ReadLine(); te.OnNext("ciao"); Console.ReadLine(); }
static void SweepCommandLine(string outputFile, string vwExe, string dataFile, string outModelDir, int numProcs = 30) { var bags = new[] { 1, 2, 4, 6, 8, 10 }.Select(a => "--bag " + a); var softmaxes = new[] { 0, 1, 2, 4, 8, 16 }.Select(a => "--softmax --lambda " + a); var epsilons = new[] { .2f }.Select(a => "--epsilon " + a); var covers = new[] { 1, 2, 4, 6, 8, 10 }.Select(a => "--cover " + a); var arguments = Util.Expand( epsilons.Union(bags).Union(softmaxes).Union(covers), new[] { "--cb_type ips", "--cb_type mtr", "--cb_type dr" }, new[] { "--marginal KG", "--marginal G", "--marginal K", "" }, new[] { 0.0002, 0.005, 0.01, 0.1 }.Select(l => string.Format(CultureInfo.InvariantCulture, "-l {0}", l)) ) .Select((a, i) => $"--cb_explore_adf --ignore B --ignore C --ignore D --ignore E --ignore F --ignore H --ignore R -b 18 --power_t 0 {a} -d {dataFile} -f {outModelDir}\\{i}.model -c") .ToList(); Directory.CreateDirectory(outModelDir); File.WriteAllLines(outputFile, arguments); int numFinishedProcessing = 0; var inputBlock = new TransformBlock <int, int>(i => i); var processBlock = new ActionBlock <int>(i => { var startTime = DateTime.UtcNow; var p = Process.Start(new ProcessStartInfo { FileName = vwExe, Arguments = arguments[i], RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }); string output = p.StandardOutput.ReadToEnd(); string error = p.StandardError.ReadToEnd(); File.WriteAllText($"{outModelDir}\\{i}.output", $"{startTime}\r\n{arguments[i]}\r\n{output}\r\n{error}"); p.WaitForExit(); int numFinished = Interlocked.Increment(ref numFinishedProcessing); Console.WriteLine($"Finished: {numFinishedProcessing} / {arguments.Count}"); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = numProcs, BoundedCapacity = numProcs }); inputBlock.LinkTo(processBlock, new DataflowLinkOptions { PropagateCompletion = true }); var input = inputBlock.AsObserver(); for (int i = 0; i < arguments.Count; i++) { input.OnNext(i); } input.OnCompleted(); processBlock.Completion.Wait(); }