public void Execute(string[] args)
{
using (var controller = Naiad.NewController.FromArgs(ref args))
{
using (var graph = controller.NewGraph())
{
var keyvals = new BatchedDataSource <Pair <string, string> >();
var queries = new BatchedDataSource <string>();
graph.NewInput(keyvals)
.KeyValueLookup(graph.NewInput(queries))
.Subscribe(list => { foreach (var l in list)
{
Console.WriteLine("value[\"{0}\"]:\t\"{1}\"", l.v1, l.v2);
}
});
graph.Activate();
Console.WriteLine("Enter two strings to insert/overwrite a (key, value) pairs.");
Console.WriteLine("Enter one string to look up a key.");
// repeatedly read lines and introduce records based on their structure.
// note: it is important to advance both inputs in order to make progress.
for (var line = Console.ReadLine(); line.Length > 0; line = Console.ReadLine())
{
var split = line.Split();
if (split.Length == 1)
{
queries.OnNext(line);
keyvals.OnNext();
}
if (split.Length == 2)
{
queries.OnNext();
keyvals.OnNext(split[0].PairWith(split[1]));
}
if (split.Length > 2)
{
Console.Error.WriteLine("error: lines with three or more strings are not understood.");
}
}
keyvals.OnCompleted();
queries.OnCompleted();
graph.Join();
}
controller.Join();
}
}
static void Main(string[] args)
{
// 1. allocate a new dataflow computation.
using (var computation = NewComputation.FromArgs(ref args))
{
Console.Out.WriteLine("Proc ID: " + computation.Configuration.ProcessID);
Console.Out.Flush();
// 2. define an object which accepts input strings.
var source = new BatchedDataSource <string>();
// 3. convert the data source into a Naiad stream of strings.
var input = computation.NewInput(source);
// 4.request a notification for each batch of strings
// received.
var output = input.Subscribe(x =>
{
foreach (var line
in x)
{
Console.WriteLine(line);
}
});
Console.Out.WriteLine("Before Activate!");
Console.Out.Flush();
// 5. start the computation, fixing the structure of
// the dataflow graph.
computation.Activate();
Console.Out.WriteLine("After Activate!");
Console.Out.Flush();
// 6. read inputs from the console as long as the
// user supplies them.
for (var l = Console.ReadLine(); l.Length > 0; l
= Console.ReadLine())
{
source.OnNext(l.Split());
}
// 7. signal that the source is now complete.
source.OnCompleted();
// 8. block until all work is finished.
computation.Join();
}
}
public void Execute(string[] args)
{
// the first thing to do is to allocate a controller from args.
using (var controller = Naiad.NewController.FromArgs(ref args))
{
// controllers allocate graphs, which are where computations are defined.
using (var graph = controller.NewGraph())
{
// 1. Make a new data source, to which we will supply strings.
var source = new BatchedDataSource <string>();
// 2. Attach source, and apply count extension method.
var counts = graph.NewInput(source).StreamingCount();
// 3. Subscribe to the resulting stream with a callback to print the outputs.
counts.Subscribe(list => { foreach (var element in list)
{
Console.WriteLine(element);
}
});
// with our dataflow graph defined, we can start soliciting strings from the user.
Console.WriteLine("Start entering lines of text. An empty line will exit the program.");
Console.WriteLine("Naiad will display counts (and changes in counts) of words you type.");
graph.Activate(); // activate the execution of this graph (no new stages allowed).
// read lines of input and hand them to the input, until an empty line appears.
for (var line = Console.ReadLine(); line.Length > 0; line = Console.ReadLine())
{
source.OnNext(line.Split());
}
source.OnCompleted(); // signal the end of the input.
graph.Join(); // waits until the graph has finished executing.
}
controller.Join(); // waits until all graphs have finished executing.
}
}
public static void Main(string[] args)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
using (var controller = NewController.FromArgs(ref args))
{
Console.WriteLine("{0}\tController constructed", stopwatch.Elapsed);
InterGraphDataSink <Pair <int, int> > graph;
var useAzure = false;
if (useAzure)
{
var container = args[0];
var prefix = args[1];
#if false
graph = controller.NewAzureInterGraphStream(container, prefix, stream => stream.Select(x => x.ParseString()));
#else
#if true
graph = controller.NewNaiadAzureInterGraphStream <Pair <Int32, Int32> >(container, prefix);
#else
using (var computation = controller.NewComputation())
{
//controller.SetConsoleOut(computation.DefaultBlobContainer("naiad-output"), "output-{0}.txt");
//controller.SetConsoleError(computation.DefaultBlobContainer("naiad-output"), "error-{0}.txt");
var graphContainer = computation.DefaultBlobContainer(container);
var data = computation.ReadFromAzureBlobs(graphContainer, prefix, stream => ReadEdges(stream, Int32.MaxValue))
//.Where(x => x.First != x.Second)
//.Select(x => x.First < x.Second ? x : x.Second.PairWith(x.First))
//.Distinct()
.WriteBinaryToAzureBlobs(graphContainer, "twitter-oriented/part-{0:000}-{1:000}.bin");
computation.Activate();
computation.Join();
}
controller.Join();
return;
#endif
#endif
}
else
{
var format = args[0];
var parts = Int32.Parse(args[1]);
graph = controller.NewInterGraphStream(Enumerable.Range(0, parts), stream => stream.Distinct()
.SelectMany(x => string.Format(format, x).ReadEdges()));
}
Console.WriteLine("{0} Graph data loaded.", stopwatch.Elapsed);
// normalize the graph to remove self-loops, point from smaller id to larger, and remove duplicates.
#if true
graph = controller.NewInterGraphStream(graph, edges => edges.Where(x => x.First != x.Second)
.Select(x => x.First < x.Second ? x : x.Second.PairWith(x.First))
//.GroupBy(x => true, (k, l) => l.RandomlyRename())
.Distinct());
#endif
// symmetrize the graph, just because...
// graph = controller.NewInterGraphStream(graph, edges => edges.Select(x => x.Second.PairWith(x.First)).Concat(edges));
Console.WriteLine("{0} Graph data normalized", stopwatch.Elapsed);
// re-orient graph edges from lower undirected degree to larger undirected degree.
// graph = controller.NewInterGraphStream(graph, edges => edges.GroupBy(x => true, (k,l) => l.RenameByDegree()));
// graph = controller.NewInterGraphStream(graph, edges => edges.OrientFromLowToHighDegree());
Console.WriteLine("{0} Graph data oriented from low to high degree", stopwatch.Elapsed);
// we build two indices, one keyed on "no" attributes and one keyed on the first.
var emptyKeyIndex = graph.ToEmptyIndex(controller, x => x.First);
var denseKeyIndex = graph.ToDenseKeyIndex(controller, x => x.First, x => x.Second);
graph.Seal(); // perhaps release the memory associated with graph. probably better to make it disposable.
Console.WriteLine("{0} Relation prefix indices built", stopwatch.Elapsed);
Console.WriteLine("{0} Starting query construction", stopwatch.Elapsed);
using (var computation = controller.NewComputation())
{
// we seed the computation with a single "true" record.
var queryseed = new BatchedDataSource <bool>();
var triangles = computation.NewInput(queryseed).Distinct()
.Triangles(emptyKeyIndex, denseKeyIndex);
// optional things to comment/uncomment, based on what sort of output we would like to see.
// triangles.Where(x => false).Subscribe();
// triangles.Expand().Where(x => { Console.WriteLine("{0} Triangle observed: {1}", stopwatch.Elapsed, x); return true; });
triangles.Select(x => x.Second.Length).Aggregate(x => true, y => y, (x, y) => x + y, (k, sum) => sum, true).Subscribe(x => { foreach (var y in x)
{
Console.WriteLine("Triangles: {0}", y);
}
});
queryseed.OnNext();
computation.Activate();
computation.Sync(0);
Console.WriteLine("{0} Synchronized", stopwatch.Elapsed);
System.Threading.Thread.Sleep(1000);
Console.WriteLine("{0} Starting query execution", stopwatch.Elapsed);
queryseed.OnCompleted(true);
computation.Join();
}
controller.Join();
}
Console.WriteLine("{0} All triangles listed", stopwatch.Elapsed);
Console.Out.Close();
}