public void Execute(string[] args) { using (var controller = Naiad.NewController.FromArgs(ref args)) { using (var graph = controller.NewGraph()) { var keyvals = new BatchedDataSource <Pair <string, string> >(); var queries = new BatchedDataSource <string>(); graph.NewInput(keyvals) .KeyValueLookup(graph.NewInput(queries)) .Subscribe(list => { foreach (var l in list) { Console.WriteLine("value[\"{0}\"]:\t\"{1}\"", l.v1, l.v2); } }); graph.Activate(); Console.WriteLine("Enter two strings to insert/overwrite a (key, value) pairs."); Console.WriteLine("Enter one string to look up a key."); // repeatedly read lines and introduce records based on their structure. // note: it is important to advance both inputs in order to make progress. for (var line = Console.ReadLine(); line.Length > 0; line = Console.ReadLine()) { var split = line.Split(); if (split.Length == 1) { queries.OnNext(line); keyvals.OnNext(); } if (split.Length == 2) { queries.OnNext(); keyvals.OnNext(split[0].PairWith(split[1])); } if (split.Length > 2) { Console.Error.WriteLine("error: lines with three or more strings are not understood."); } } keyvals.OnCompleted(); queries.OnCompleted(); graph.Join(); } controller.Join(); } }
static void Main(string[] args) { // 1. allocate a new dataflow computation. using (var computation = NewComputation.FromArgs(ref args)) { Console.Out.WriteLine("Proc ID: " + computation.Configuration.ProcessID); Console.Out.Flush(); // 2. define an object which accepts input strings. var source = new BatchedDataSource <string>(); // 3. convert the data source into a Naiad stream of strings. var input = computation.NewInput(source); // 4.request a notification for each batch of strings // received. var output = input.Subscribe(x => { foreach (var line in x) { Console.WriteLine(line); } }); Console.Out.WriteLine("Before Activate!"); Console.Out.Flush(); // 5. start the computation, fixing the structure of // the dataflow graph. computation.Activate(); Console.Out.WriteLine("After Activate!"); Console.Out.Flush(); // 6. read inputs from the console as long as the // user supplies them. for (var l = Console.ReadLine(); l.Length > 0; l = Console.ReadLine()) { source.OnNext(l.Split()); } // 7. signal that the source is now complete. source.OnCompleted(); // 8. block until all work is finished. computation.Join(); } }
public void Execute(string[] args) { // the first thing to do is to allocate a controller from args. using (var controller = Naiad.NewController.FromArgs(ref args)) { // controllers allocate graphs, which are where computations are defined. using (var graph = controller.NewGraph()) { // 1. Make a new data source, to which we will supply strings. var source = new BatchedDataSource <string>(); // 2. Attach source, and apply count extension method. var counts = graph.NewInput(source).StreamingCount(); // 3. Subscribe to the resulting stream with a callback to print the outputs. counts.Subscribe(list => { foreach (var element in list) { Console.WriteLine(element); } }); // with our dataflow graph defined, we can start soliciting strings from the user. Console.WriteLine("Start entering lines of text. An empty line will exit the program."); Console.WriteLine("Naiad will display counts (and changes in counts) of words you type."); graph.Activate(); // activate the execution of this graph (no new stages allowed). // read lines of input and hand them to the input, until an empty line appears. for (var line = Console.ReadLine(); line.Length > 0; line = Console.ReadLine()) { source.OnNext(line.Split()); } source.OnCompleted(); // signal the end of the input. graph.Join(); // waits until the graph has finished executing. } controller.Join(); // waits until all graphs have finished executing. } }
public static void Main(string[] args) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); using (var controller = NewController.FromArgs(ref args)) { Console.WriteLine("{0}\tController constructed", stopwatch.Elapsed); InterGraphDataSink <Pair <int, int> > graph; var useAzure = false; if (useAzure) { var container = args[0]; var prefix = args[1]; #if false graph = controller.NewAzureInterGraphStream(container, prefix, stream => stream.Select(x => x.ParseString())); #else #if true graph = controller.NewNaiadAzureInterGraphStream <Pair <Int32, Int32> >(container, prefix); #else using (var computation = controller.NewComputation()) { //controller.SetConsoleOut(computation.DefaultBlobContainer("naiad-output"), "output-{0}.txt"); //controller.SetConsoleError(computation.DefaultBlobContainer("naiad-output"), "error-{0}.txt"); var graphContainer = computation.DefaultBlobContainer(container); var data = computation.ReadFromAzureBlobs(graphContainer, prefix, stream => ReadEdges(stream, Int32.MaxValue)) //.Where(x => x.First != x.Second) //.Select(x => x.First < x.Second ? x : x.Second.PairWith(x.First)) //.Distinct() .WriteBinaryToAzureBlobs(graphContainer, "twitter-oriented/part-{0:000}-{1:000}.bin"); computation.Activate(); computation.Join(); } controller.Join(); return; #endif #endif } else { var format = args[0]; var parts = Int32.Parse(args[1]); graph = controller.NewInterGraphStream(Enumerable.Range(0, parts), stream => stream.Distinct() .SelectMany(x => string.Format(format, x).ReadEdges())); } Console.WriteLine("{0} Graph data loaded.", stopwatch.Elapsed); // normalize the graph to remove self-loops, point from smaller id to larger, and remove duplicates. #if true graph = controller.NewInterGraphStream(graph, edges => edges.Where(x => x.First != x.Second) .Select(x => x.First < x.Second ? x : x.Second.PairWith(x.First)) //.GroupBy(x => true, (k, l) => l.RandomlyRename()) .Distinct()); #endif // symmetrize the graph, just because... // graph = controller.NewInterGraphStream(graph, edges => edges.Select(x => x.Second.PairWith(x.First)).Concat(edges)); Console.WriteLine("{0} Graph data normalized", stopwatch.Elapsed); // re-orient graph edges from lower undirected degree to larger undirected degree. // graph = controller.NewInterGraphStream(graph, edges => edges.GroupBy(x => true, (k,l) => l.RenameByDegree())); // graph = controller.NewInterGraphStream(graph, edges => edges.OrientFromLowToHighDegree()); Console.WriteLine("{0} Graph data oriented from low to high degree", stopwatch.Elapsed); // we build two indices, one keyed on "no" attributes and one keyed on the first. var emptyKeyIndex = graph.ToEmptyIndex(controller, x => x.First); var denseKeyIndex = graph.ToDenseKeyIndex(controller, x => x.First, x => x.Second); graph.Seal(); // perhaps release the memory associated with graph. probably better to make it disposable. Console.WriteLine("{0} Relation prefix indices built", stopwatch.Elapsed); Console.WriteLine("{0} Starting query construction", stopwatch.Elapsed); using (var computation = controller.NewComputation()) { // we seed the computation with a single "true" record. var queryseed = new BatchedDataSource <bool>(); var triangles = computation.NewInput(queryseed).Distinct() .Triangles(emptyKeyIndex, denseKeyIndex); // optional things to comment/uncomment, based on what sort of output we would like to see. // triangles.Where(x => false).Subscribe(); // triangles.Expand().Where(x => { Console.WriteLine("{0} Triangle observed: {1}", stopwatch.Elapsed, x); return true; }); triangles.Select(x => x.Second.Length).Aggregate(x => true, y => y, (x, y) => x + y, (k, sum) => sum, true).Subscribe(x => { foreach (var y in x) { Console.WriteLine("Triangles: {0}", y); } }); queryseed.OnNext(); computation.Activate(); computation.Sync(0); Console.WriteLine("{0} Synchronized", stopwatch.Elapsed); System.Threading.Thread.Sleep(1000); Console.WriteLine("{0} Starting query execution", stopwatch.Elapsed); queryseed.OnCompleted(true); computation.Join(); } controller.Join(); } Console.WriteLine("{0} All triangles listed", stopwatch.Elapsed); Console.Out.Close(); }