/// <summary> /// Executes a word counting Naiad program. /// </summary> /// <param name="config">Naiad controller configuration</param> /// <param name="args">Remaining arguments</param> public void Execute(string[] args) { // first, construct a Naiad controller. using (var controller = NewController.FromArgs(ref args)) { using (var graph = controller.NewGraph()) { // create an incrementally updateable collection var text = new IncrementalCollection <string>(graph);//.NewInput<string>(); // segment strings, count, and print text.SelectMany(x => x.Split(' ')) .Count(y => y, (k, c) => k + ":" + c) // yields "word:count" for each word .Subscribe(l => { foreach (var element in l) { Console.WriteLine(element); } }); graph.Activate(); Console.WriteLine("Start entering lines of text. An empty line will exit the program."); Console.WriteLine("Naiad will display counts (and changes in counts) of words you type."); var line = Console.ReadLine(); for (int i = 0; line != ""; i++) { text.OnNext(line); graph.Sync(i); line = Console.ReadLine(); } text.OnCompleted(); // closes input graph.Join(); } controller.Join(); // blocks until flushed } }
public void Execute(string[] args) { int documentCount = 100000; int vocabulary = 100000; int batchSize = 10000; int iterations = 10; using (var controller = NewController.FromArgs(ref args)) { #region building up input data if (args.Length == 5) { documentCount = Convert.ToInt32(args[1]); vocabulary = Convert.ToInt32(args[2]); batchSize = Convert.ToInt32(args[3]); iterations = Convert.ToInt32(args[4]); } var random = new Random(0); List <Document> docs = Enumerable.Range(0, documentCount) .Select(i => new Document(Enumerable.Range(0, 10) .Select(j => String.Format("{0}", random.Next(vocabulary))) .Aggregate((x, y) => x + " " + y), i)).ToList <Document>(); List <Query>[] queryBatches = new List <Query> [iterations]; for (int i = 0; i < iterations; i++) { queryBatches[i] = Enumerable.Range(i * batchSize, batchSize) .Select(j => new Query(String.Format("{0}", j % vocabulary), j, 1)) .ToList(); } #endregion using (var manager = controller.NewGraph()) { // declare inputs for documents and queries. var documents = new IncrementalCollection <Document>(manager); var queries = new IncrementalCollection <Query>(manager); // each document is broken down into a collection of terms, each with associated identifier. var dTerms = documents.SelectMany(doc => doc.text.Split(' ').Select(term => new Document(term, doc.id))) .Distinct(); // each query is broken down into a collection of terms, each with associated identifier and threshold. var qTerms = queries.SelectMany(query => query.text.Split(' ').Select(term => new Query(term, query.id, query.threshold))) .Distinct(); // doc terms and query terms are joined, matching pairs are counted and returned if the count exceeds the threshold. var results = dTerms.Join(qTerms, d => d.text, q => q.text, (d, q) => new Match(d.id, q.id, q.threshold)) .Count(match => match) .Select(pair => new Match(pair.v1.document, pair.v1.query, pair.v1.threshold - (int)pair.v2)) .Where(match => match.threshold <= 0) .Select(match => new Pair <int, int>(match.document, match.query)); // subscribe to the output in case we are interested in the results var subscription = results.Subscribe(list => Console.WriteLine("matches found: {0}", list.Length)); manager.Activate(); #region Prepare some fake documents to put in the collection // creates many documents each containing 10 words from [0, ... vocabulary-1]. int share_size = docs.Count / controller.Configuration.Processes; documents.OnNext(docs.GetRange(controller.Configuration.ProcessID * share_size, share_size)); queries.OnNext(); //Console.WriteLine("Example SearchIndex in Naiad. Step 1: indexing documents, step 2: issuing queries."); Console.WriteLine("Indexing {0} random documents, {1} terms (please wait)", documentCount, 10 * documentCount); subscription.Sync(0); #endregion #region Issue batches of queries and assess performance Console.WriteLine("Issuing {0} rounds of batches of {1} queries (press [enter] to start)", iterations, batchSize); Console.ReadLine(); var stopwatch = System.Diagnostics.Stopwatch.StartNew(); for (int i = 0; i < iterations; i++) { // we round-robin through query terms. more advanced queries are possible. if (controller.Configuration.ProcessID == 0) { queries.OnNext(queryBatches[i]); // introduce new queries. } else { queries.OnNext(); } documents.OnNext(); // indicate no new docs. subscription.Sync(i + 1); // block until round is done. } documents.OnCompleted(); queries.OnCompleted(); controller.Join(); #endregion manager.Join(); } controller.Join(); } }