Stream <string, Epoch> loadDiskFiles(OneOffComputation computation, String inputDir, int numFiles) { var processes = computation.Configuration.Processes; var thisProcess = computation.Configuration.ProcessID; List <string> myFiles = new List <string>(); bool printFirstInput = false; for (int i = 0; i < numFiles; i++) { // ensure we generate the same graph no matter how many processes there are if ((i % processes) == thisProcess) { myFiles.Add(inputDir + getFileName(i, numFiles)); if (!printFirstInput) { Console.Out.WriteLine("my process ID: " + thisProcess.ToString() + ", and my first file is " + inputDir + i.ToString()); printFirstInput = true; } } } var text = myFiles.ToArray() .AsNaiadStream(computation) .SelectMany(x => x.ReadLinesOfText()); //.SelectMany(x => x.ReadLinesOfText()); return(text); }
public NaiadSolution(string[] args) { rawUsers = new List <User>(); rawPosts = new List <Post>(); rawComments = new List <Comment>(); rawCommentedEdges = new List <CommentedEdge>(); rawLikesEdges = new List <LikesEdge>(); rawPostEdges = new List <PostEdge>(); rawSubmitterEdges = new List <SubmitterEdge>(); rawFriendEdges = new List <FriendEdge>(); computation = NewComputation.FromArgs(ref args); users = computation.NewInputCollection <User>(); posts = computation.NewInputCollection <Post>(); comments = computation.NewInputCollection <Comment>(); commentedEdges = computation.NewInputCollection <CommentedEdge>(); likesEdges = computation.NewInputCollection <LikesEdge>(); postEdges = computation.NewInputCollection <PostEdge>(); submitterEdges = computation.NewInputCollection <SubmitterEdge>(); friendEdges = computation.NewInputCollection <FriendEdge>(); actualEpoch = -1; isDisposed = false; }
public void Execute(string[] args) { using (OneOffComputation computation = NewComputation.FromArgs(ref args)) { int numToExchange = args.Length > 1 ? int.Parse(args[1]) : 1000000; Stream <int, Epoch> input = computation.NewInput(new ConstantDataSource <int>(5)); Stream <int, Epoch> stream = ProducerVertex.MakeStage(numToExchange, computation.Configuration.WorkerCount, input); Stage <ConsumerVertex, Epoch> consumer = ConsumerVertex.MakeStage(numToExchange, computation.Configuration.WorkerCount, stream); computation.Activate(); computation.Join(); } }
public void Execute(string[] args) { using (OneOffComputation computation = NewComputation.FromArgs(ref args)) { int numToExchange = args.Length > 1 ? int.Parse(args[1]) : 1000000; int producers = Int32.Parse(args[2]); int consumers = Int32.Parse(args[3]); var exchange = args.Length > 4 && args[4] == "exchange"; var input = new Pair <int, int>[] { }.AsNaiadStream(computation); Stream <Pair <int, int>, Epoch> stream = ProducerVertex.MakeStage(numToExchange, 0, producers, computation.Configuration.WorkerCount, input); Stage <ConsumerVertex, Epoch> consumer = ConsumerVertex.MakeStage(numToExchange, computation.Configuration.Processes - consumers, computation.Configuration.Processes, computation.Configuration.WorkerCount, exchange, stream); computation.Activate(); computation.Join(); } }
Stream <Pair <Byte[], Byte[]>, Epoch> loadDiskFiles(OneOffComputation computation, String inputDir, int numFiles) { var processes = computation.Configuration.Processes; var thisProcess = computation.Configuration.ProcessID; List <string> myFiles = new List <string>(); for (int i = 0; i < numFiles; i++) { // ensure we generate the same graph no matter how many processes there are if ((i % processes) == thisProcess) { myFiles.Add(inputDir + getFileName(i, numFiles)); Console.Out.WriteLine("my process ID: " + thisProcess.ToString() + ", and my files have " + inputDir + i.ToString()); } } var text = myFiles.ToArray() .AsNaiadStream(computation) .Synchronize(x => true) .SelectMany(x => x.ReadLinesOfKeyValue(KEY_LEN, VALUE_LEN)); //.SelectMany(x => x.ReadLinesOfText()); return(text); }