Stream <string, Epoch> loadDiskFiles(OneOffComputation computation, String inputDir, int numFiles)
{
var processes = computation.Configuration.Processes;
var thisProcess = computation.Configuration.ProcessID;
List <string> myFiles = new List <string>();
bool printFirstInput = false;
for (int i = 0; i < numFiles; i++)
{
// ensure we generate the same graph no matter how many processes there are
if ((i % processes) == thisProcess)
{
myFiles.Add(inputDir + getFileName(i, numFiles));
if (!printFirstInput)
{
Console.Out.WriteLine("my process ID: " + thisProcess.ToString() + ", and my first file is " + inputDir + i.ToString());
printFirstInput = true;
}
}
}
var text = myFiles.ToArray()
.AsNaiadStream(computation)
.SelectMany(x => x.ReadLinesOfText());
//.SelectMany(x => x.ReadLinesOfText());
return(text);
}
public NaiadSolution(string[] args)
{
rawUsers = new List <User>();
rawPosts = new List <Post>();
rawComments = new List <Comment>();
rawCommentedEdges = new List <CommentedEdge>();
rawLikesEdges = new List <LikesEdge>();
rawPostEdges = new List <PostEdge>();
rawSubmitterEdges = new List <SubmitterEdge>();
rawFriendEdges = new List <FriendEdge>();
computation = NewComputation.FromArgs(ref args);
users = computation.NewInputCollection <User>();
posts = computation.NewInputCollection <Post>();
comments = computation.NewInputCollection <Comment>();
commentedEdges = computation.NewInputCollection <CommentedEdge>();
likesEdges = computation.NewInputCollection <LikesEdge>();
postEdges = computation.NewInputCollection <PostEdge>();
submitterEdges = computation.NewInputCollection <SubmitterEdge>();
friendEdges = computation.NewInputCollection <FriendEdge>();
actualEpoch = -1;
isDisposed = false;
}
public void Execute(string[] args)
{
using (OneOffComputation computation = NewComputation.FromArgs(ref args))
{
int numToExchange = args.Length > 1 ? int.Parse(args[1]) : 1000000;
Stream <int, Epoch> input = computation.NewInput(new ConstantDataSource <int>(5));
Stream <int, Epoch> stream = ProducerVertex.MakeStage(numToExchange, computation.Configuration.WorkerCount, input);
Stage <ConsumerVertex, Epoch> consumer = ConsumerVertex.MakeStage(numToExchange, computation.Configuration.WorkerCount, stream);
computation.Activate();
computation.Join();
}
}
public void Execute(string[] args)
{
using (OneOffComputation computation = NewComputation.FromArgs(ref args))
{
int numToExchange = args.Length > 1 ? int.Parse(args[1]) : 1000000;
int producers = Int32.Parse(args[2]);
int consumers = Int32.Parse(args[3]);
var exchange = args.Length > 4 && args[4] == "exchange";
var input = new Pair <int, int>[] { }.AsNaiadStream(computation);
Stream <Pair <int, int>, Epoch> stream = ProducerVertex.MakeStage(numToExchange, 0, producers, computation.Configuration.WorkerCount, input);
Stage <ConsumerVertex, Epoch> consumer = ConsumerVertex.MakeStage(numToExchange, computation.Configuration.Processes - consumers, computation.Configuration.Processes, computation.Configuration.WorkerCount, exchange, stream);
computation.Activate();
computation.Join();
}
}
Stream <Pair <Byte[], Byte[]>, Epoch> loadDiskFiles(OneOffComputation computation, String inputDir, int numFiles)
{
var processes = computation.Configuration.Processes;
var thisProcess = computation.Configuration.ProcessID;
List <string> myFiles = new List <string>();
for (int i = 0; i < numFiles; i++)
{
// ensure we generate the same graph no matter how many processes there are
if ((i % processes) == thisProcess)
{
myFiles.Add(inputDir + getFileName(i, numFiles));
Console.Out.WriteLine("my process ID: " + thisProcess.ToString() + ", and my files have " + inputDir + i.ToString());
}
}
var text = myFiles.ToArray()
.AsNaiadStream(computation)
.Synchronize(x => true)
.SelectMany(x => x.ReadLinesOfKeyValue(KEY_LEN, VALUE_LEN));
//.SelectMany(x => x.ReadLinesOfText());
return(text);
}