/* Parse given input file into tupleBuffer */
private static void readTuplesFromFile(string filepath, TupleIdGenerator tupleIdGenerator)
{
TextReader freader;
string line;
try {
freader = File.OpenText(filepath);
} catch (Exception e) {
Console.WriteLine("[ERROR] Unable to Open the file: \"{0}\". Proceding without it. {1}", filepath, e.Message);
return;
}
Console.WriteLine("[Operator] Reading from file: {0}", filepath);
while ((line = freader.ReadLine()) != null)
{
if (line.StartsWith("%")) // ignore comments
{
continue;
}
var tuple = line.Split(',').ToList();
for (int i = 0; i < tuple.Count; i++)
{
tuple[i] = tuple[i].Trim(' '); // strip whitespaces
}
thisReplica.Flow(new DadTuple(tupleIdGenerator.NextTupleId(), tuple));
}
}
public TupleProcessor(AbstractKernel kernel, List <DownstreamOperator> downstreamOperators, TupleIdGenerator tupleIdGenerator, GroupManager group)
{
TupleBuffer = new BufferBlock <DadTuple>();
Kernel = kernel;
DownstreamOperators = downstreamOperators;
TupleIdGenerator = tupleIdGenerator;
Group = group;
outputFilePath = GetOutputFileName();
}
private void Process(DadTuple inputTuple)
{
Console.WriteLine("[CONSUMER] Starting computation of {0}.", inputTuple);
// Process the tuple
var computedTuples = Kernel.execute(inputTuple.Content);
if (computedTuples.Count == 0)
{
return;
}
var outTuples = computedTuples.Select(t => new DadTuple(TupleIdGenerator.NextTupleId(), t)).ToList();
Console.WriteLine("[CONSUMER] Sharing results of computation of {0} to group.", inputTuple);
Group.RMSend(p => p.SaveProcessedTuples(inputTuple.Id, outTuples));
//Group.LocalReplica.SaveProcessedTuples(inputTuple.Id, outTuples);
// Is the last operator in the acyclic graph
if (DownstreamOperators.Count == 0)
{
// Output to a file (if we are the leader)
OutputToFile(outTuples);
}
else
{
// Send the tuples
SendOutputTuples(outTuples);
}
// Log
LogToPuppetMaster(outTuples);
Group.RMSend(p => p.DeliveredTuples(inputTuple.Id, outTuples.Select(ot => ot.Id).ToList()));
}
public static void Main(string[] args)
{
// Check args validity
if (args.Length != 3)
{
Console.WriteLine("[Operator] Usage: OperatorExecutable <serialized_operator> <replicaID>!");
Console.WriteLine("[Operator] Received: ");
foreach (string s in args)
{
Console.WriteLine(s);
}
exiting(5);
return;
}
// Parse operator
string serializedOperator = args[0];
Operator operatorData;
try {
operatorData = Operator.deserialize(serializedOperator);
} catch (Exception e) {
Console.WriteLine("[Operator] Couldn't deserialize the operator. Cause: {0}", e.Message);
Console.WriteLine("[Operator] Received: {0}", serializedOperator);
exiting(10);
return;
}
// Parse replicaId
int replicaID;
try {
replicaID = int.Parse(args[1]);
if (replicaID >= operatorData.replicaURLs.Count)
{
throw new Exception();
}
} catch (Exception) {
Console.WriteLine("[REPLICA] ReplicaID was not a valid or out of range integer!");
exiting(5);
return;
}
myURL = operatorData.replicaURLs[replicaID];
Console.WriteLine("------------------------------------------------------------------");
Console.WriteLine("[REPLICA] Operator: {0}({1})! URL: {2}", operatorData.id, replicaID, myURL);
Console.WriteLine("------------------------------------------------------------------");
// initializeOperatorSpec
var kernel = initializeOperatorSpec(operatorData);
var downstreamOperators = SetUpDownstream(operatorData);
// Connect to the puppet master
string puppetMasterLoggerUrl = args[2];
try {
PuppetMaster = (IPuppetMaster)Activator.GetObject(typeof(IPuppetMaster), puppetMasterLoggerUrl);
Console.WriteLine("[Replica] Created proxy for puppetMaster at {0}", puppetMasterLoggerUrl);
PuppetMaster.log("Replica " + operatorData.id + " of operator " + myURL + " launched.");
} catch (Exception e) {
Console.WriteLine("[Operator] Unable to talk to the puppetMaster at: {0}. Cause: {1}", puppetMasterLoggerUrl, e.Message);
}
//System.Diagnostics.Debugger.Launch();
var localRoutingStrategy = RoutingPolicyToStrategy(operatorData.routingPolicy, operatorData.HashingField);
var localReplica = new LocalReplica(operatorData.id, replicaID, null);
var tupleIdGenerator = new TupleIdGenerator(operatorData.id, replicaID);
var monitoringProcess = new MonitoringProcess();
var group = new GroupManager(SetUpGroup(operatorData, replicaID, localReplica), localReplica, monitoringProcess, localRoutingStrategy);
var tupleProcessor = new TupleProcessor(kernel, downstreamOperators, tupleIdGenerator, group);
thisReplica = new ReplicaServices(tupleProcessor, group, monitoringProcess, operatorData, operatorData.id, replicaID);
localReplica.SetProxy(thisReplica);
// Publish the service
string replicaURL = operatorData.replicaURLs[replicaID];
string serviceName = getNameFromURL(replicaURL);
try {
int servicePort = getPortFromURL(replicaURL);
TcpChannel channel = new TcpChannel(servicePort);
// @SEE: I think because the PCS has already registered a channel and this processe is a child of it we can´t register again?
//ChannelServices.RegisterChannel(channel, false); // @SEE: should be true??
Type serviceType = typeof(ReplicaServices);
RemotingServices.Marshal(thisReplica, serviceName, serviceType);
System.Console.WriteLine("Started Operator with name {0} at port {1}.", serviceName, servicePort);
}
catch (Exception e) {
Console.WriteLine("[Operator] Unable to publish {0}. Cause: {1}", replicaURL, e.Message);
exiting(20);
return;
}
// See if we need to read from a file
foreach (string fileName in operatorData.inputFiles)
{
readTuplesFromFile(INPUT_FOLDER_PATH + fileName, tupleIdGenerator);
}
Console.ReadLine();
}