public void TestSwitch()
{
var input = new[] { 1, 2, 3, 4, 5 };
Predicate <int> predicate = i => i % 2 == 0;
var target1 = new BufferBlock <int>();
var target2 = new BufferBlock <int>();
var inputBlock = new TransformBlock <int, int>(i => i);
inputBlock
.LinkWhen(predicate, target1)
.LinkOtherwise(target2);
input.ToObservable()
.Subscribe(inputBlock.AsObserver());
IList <int> output1 = target1
.AsObservable()
.ToEnumerable()
.ToList();
IList <int> output2 = target2
.AsObservable()
.ToEnumerable()
.ToList();
output1.Should()
.BeEquivalentTo(input.Where(i => predicate(i)));
output2.Should()
.BeEquivalentTo(input.Where(i => !predicate(i)));
}
public static void Transform(string fileIn, string fileOut, Func <JsonTextReader, JsonTextWriter, bool> transform)
{
using (var reader = new StreamReader(fileIn, Encoding.UTF8))
using (var writer = new StreamWriter(fileOut, false, Encoding.UTF8))
{
var transformBlock = new TransformBlock <string, string>(
evt =>
{
var stringWriter = new StringWriter();
using (var jsonReader = new JsonTextReader(new StringReader(evt)))
using (var jsonWriter = new JsonTextWriter(stringWriter))
{
while (jsonReader.Read())
{
if (!transform(jsonReader, jsonWriter))
{
jsonWriter.WriteToken(jsonReader.TokenType, jsonReader.Value);
}
}
}
return(stringWriter.ToString());
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1024,
MaxDegreeOfParallelism = 8 // TODO:parameterize
});
var outputBock = new ActionBlock <string>(l => { if (!string.IsNullOrEmpty(l))
{
writer.WriteLine(l);
}
},
new ExecutionDataflowBlockOptions {
BoundedCapacity = 1024, MaxDegreeOfParallelism = 1
});
transformBlock.LinkTo(outputBock, new DataflowLinkOptions {
PropagateCompletion = true
});
var input = transformBlock.AsObserver();
string line;
while ((line = reader.ReadLine()) != null)
{
input.OnNext(line);
}
input.OnCompleted();
outputBock.Completion.Wait();
}
}
public static ISourceBlock <List <T> > CreateBatch <T>(string file, Func <Line, T> transform)
{
var block = new TransformBlock <List <Line>, List <T> >(
batch => batch.Select(transform).ToList(),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 8,
BoundedCapacity = 128
});
new Thread(() =>
{
var input = block.AsObserver();
try
{
using (var reader = new StreamReader(file))
{
string line;
int lineNr = 0;
var batch = new List <Line>();
while ((line = reader.ReadLine()) != null)
{
batch.Add(new Line {
Content = line, Number = lineNr++
});
if (batch.Count >= 512)
{
input.OnNext(batch);
batch = new List <Line>();
}
}
if (batch.Count > 0)
{
input.OnNext(batch);
}
input.OnCompleted();
}
}
catch (Exception ex)
{
input.OnError(ex);
}
}).Start();
return(block);
// var order = new TransformBlock<List<T>, List<T>>(t => t, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
// block.LinkTo(order, new DataflowLinkOptions { PropagateCompletion = true });
// return order;
}
public static ISourceBlock <T> Create <T>(string file, Func <Line, T> transform)
{
var block = new TransformBlock <Line, T>(
transform,
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 2,
BoundedCapacity = 128
});
new Thread(() =>
{
var input = block.AsObserver();
try
{
using (var reader = new StreamReader(file))
{
string line;
int lineNr = 0;
var batch = new List <Line>();
while ((line = reader.ReadLine()) != null)
{
input.OnNext(new Line {
Content = line, Number = lineNr++
});
}
input.OnCompleted();
}
}
catch (Exception ex)
{
input.OnError(ex);
}
}).Start();
return(block);
}
public static void Convert(StreamReader reader, StreamWriter writer)
{
var line = reader.ReadLine();
if (line == null)
{
return;
}
var jExample = JObject.Parse(line);
var settings = jExample.Properties().Any(p => p.Name == "_multi") ? "--cb_explore_adf" : "--cb_explore";
int lineNr = 1;
using (var vw = new VowpalWabbit(new VowpalWabbitSettings(settings)
{
EnableStringExampleGeneration = true,
EnableStringFloatCompact = true,
EnableThreadSafeExamplePooling = true
}))
{
var serializeBlock = new TransformBlock <Tuple <string, int>, string>(l =>
{
using (var jsonSerializer = new VowpalWabbitJsonSerializer(vw))
using (var example = jsonSerializer.ParseAndCreate(l.Item1))
{
if (example == null)
{
throw new InvalidDataException($"Invalid example in line {l.Item2}: '{l.Item1}'");
}
var str = example.VowpalWabbitString;
if (example is VowpalWabbitMultiLineExampleCollection)
{
str += "\n";
}
return(str);
}
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1024,
MaxDegreeOfParallelism = 8
});
var writeBlock = new ActionBlock <string>(
l => writer.WriteLine(l),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1, BoundedCapacity = 128
});
serializeBlock.LinkTo(writeBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
var input = serializeBlock.AsObserver();
do
{
input.OnNext(Tuple.Create(line, lineNr));
lineNr++;
} while ((line = reader.ReadLine()) != null);
input.OnCompleted();
serializeBlock.Completion.Wait();
}
}
static void Main(string[] args)
{
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock<string, string>(uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return new WebClient().DownloadString(uri);
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock<string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character to.
char[] tokens = text.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
tokens[i] = ' ';
}
text = new string(tokens);
// Separate the text into an array of words.
return text.Split(new char[] { ' ' },
StringSplitOptions.RemoveEmptyEntries);
});
// Removes short words, orders the resulting words alphabetically,
// and then remove duplicates.
var filterWordList = new TransformBlock<string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return words.Where(word => word.Length > 3).OrderBy(word => word)
.Distinct().ToArray();
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock<string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
// Holds reversed words.
var reversedWords = new ConcurrentQueue<string>();
// Add each word in the original collection to the result whose
// reversed word also exists in the collection.
Parallel.ForEach(words, word =>
{
// Reverse the work.
string reverse = new string(word.Reverse().ToArray());
// Enqueue the word if the reversed version also exists
// in the collection.
if (Array.BinarySearch<string>(words, reverse) >= 0 &&
word != reverse)
{
reversedWords.Enqueue(word);
}
});
return reversedWords;
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock<string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
//
// Connect the dataflow blocks to form a pipeline.
//
downloadString.AsObservable().Subscribe(i =>
Console.WriteLine("Action Sub {0}", i));
var te= downloadString.AsObserver();
downloadString.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(findReversedWords);
findReversedWords.LinkTo(printReversedWords);
//
// For each completion task in the pipeline, create a continuation task
// that marks the next block in the pipeline as completed.
// A completed dataflow block processes any buffered elements, but does
// not accept new elements.
//
downloadString.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)createWordList).Fault(t.Exception);
else createWordList.Complete();
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterWordList).Fault(t.Exception);
else filterWordList.Complete();
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)findReversedWords).Fault(t.Exception);
else findReversedWords.Complete();
});
findReversedWords.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)printReversedWords).Fault(t.Exception);
else printReversedWords.Complete();
});
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
Console.ReadLine();
te.OnNext("ciao");
Console.ReadLine();
}
static void Main(string[] args)
{
//
// Create the members of the pipeline.
//
//var nums = Enumerable.Range(0, 10).ToList();//.AsQueryable();
//var query = (from num in nums.AsQueryable()
// where num % 2 == 0
// select num * num).Sum();
//int sum = 0;
//for (int index = 0; index < nums.Count; index++)
//{
// int num = nums[index];
// if (num % 2 == 0)
// sum += num * num;
//}
// Downloads the requested resource as a string.
var downloadString = new TransformBlock <string, string>(uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return(new WebClient().DownloadString(uri));
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character to.
char[] tokens = text.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
{
tokens[i] = ' ';
}
}
text = new string(tokens);
// Separate the text into an array of words.
return(text.Split(new char[] { ' ' },
StringSplitOptions.RemoveEmptyEntries));
});
// Removes short words, orders the resulting words alphabetically,
// and then remove duplicates.
var filterWordList = new TransformBlock <string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return(words.Where(word => word.Length > 3).OrderBy(word => word)
.Distinct().ToArray());
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock <string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
// Holds reversed words.
var reversedWords = new ConcurrentQueue <string>();
// Add each word in the original collection to the result whose
// reversed word also exists in the collection.
Parallel.ForEach(words, word =>
{
// Reverse the work.
string reverse = new string(word.Reverse().ToArray());
// Enqueue the word if the reversed version also exists
// in the collection.
if (Array.BinarySearch <string>(words, reverse) >= 0 &&
word != reverse)
{
reversedWords.Enqueue(word);
}
});
return(reversedWords);
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock <string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
//
// Connect the dataflow blocks to form a pipeline.
//
downloadString.AsObservable().Subscribe(i =>
Console.WriteLine("Action Sub {0}", i));
var te = downloadString.AsObserver();
downloadString.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(findReversedWords);
findReversedWords.LinkTo(printReversedWords);
//
// For each completion task in the pipeline, create a continuation task
// that marks the next block in the pipeline as completed.
// A completed dataflow block processes any buffered elements, but does
// not accept new elements.
//
downloadString.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)createWordList).Fault(t.Exception);
}
else
{
createWordList.Complete();
}
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)filterWordList).Fault(t.Exception);
}
else
{
filterWordList.Complete();
}
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)findReversedWords).Fault(t.Exception);
}
else
{
findReversedWords.Complete();
}
});
findReversedWords.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)printReversedWords).Fault(t.Exception);
}
else
{
printReversedWords.Complete();
}
});
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
Console.ReadLine();
te.OnNext("ciao");
Console.ReadLine();
}
static void SweepCommandLine(string outputFile, string vwExe, string dataFile, string outModelDir, int numProcs = 30)
{
var bags = new[] { 1, 2, 4, 6, 8, 10 }.Select(a => "--bag " + a);
var softmaxes = new[] { 0, 1, 2, 4, 8, 16 }.Select(a => "--softmax --lambda " + a);
var epsilons = new[] { .2f }.Select(a => "--epsilon " + a);
var covers = new[] { 1, 2, 4, 6, 8, 10 }.Select(a => "--cover " + a);
var arguments = Util.Expand(
epsilons.Union(bags).Union(softmaxes).Union(covers),
new[] { "--cb_type ips", "--cb_type mtr", "--cb_type dr" },
new[] { "--marginal KG", "--marginal G", "--marginal K", "" },
new[] { 0.0002, 0.005, 0.01, 0.1 }.Select(l => string.Format(CultureInfo.InvariantCulture, "-l {0}", l))
)
.Select((a, i) => $"--cb_explore_adf --ignore B --ignore C --ignore D --ignore E --ignore F --ignore H --ignore R -b 18 --power_t 0 {a} -d {dataFile} -f {outModelDir}\\{i}.model -c")
.ToList();
Directory.CreateDirectory(outModelDir);
File.WriteAllLines(outputFile, arguments);
int numFinishedProcessing = 0;
var inputBlock = new TransformBlock <int, int>(i => i);
var processBlock = new ActionBlock <int>(i =>
{
var startTime = DateTime.UtcNow;
var p = Process.Start(new ProcessStartInfo
{
FileName = vwExe,
Arguments = arguments[i],
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
});
string output = p.StandardOutput.ReadToEnd();
string error = p.StandardError.ReadToEnd();
File.WriteAllText($"{outModelDir}\\{i}.output", $"{startTime}\r\n{arguments[i]}\r\n{output}\r\n{error}");
p.WaitForExit();
int numFinished = Interlocked.Increment(ref numFinishedProcessing);
Console.WriteLine($"Finished: {numFinishedProcessing} / {arguments.Count}");
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = numProcs, BoundedCapacity = numProcs
});
inputBlock.LinkTo(processBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
var input = inputBlock.AsObserver();
for (int i = 0; i < arguments.Count; i++)
{
input.OnNext(i);
}
input.OnCompleted();
processBlock.Completion.Wait();
}
