/* Parse given input file into tupleBuffer */ private static void readTuplesFromFile(string filepath, TupleIdGenerator tupleIdGenerator) { TextReader freader; string line; try { freader = File.OpenText(filepath); } catch (Exception e) { Console.WriteLine("[ERROR] Unable to Open the file: \"{0}\". Proceding without it. {1}", filepath, e.Message); return; } Console.WriteLine("[Operator] Reading from file: {0}", filepath); while ((line = freader.ReadLine()) != null) { if (line.StartsWith("%")) // ignore comments { continue; } var tuple = line.Split(',').ToList(); for (int i = 0; i < tuple.Count; i++) { tuple[i] = tuple[i].Trim(' '); // strip whitespaces } thisReplica.Flow(new DadTuple(tupleIdGenerator.NextTupleId(), tuple)); } }
public TupleProcessor(AbstractKernel kernel, List <DownstreamOperator> downstreamOperators, TupleIdGenerator tupleIdGenerator, GroupManager group) { TupleBuffer = new BufferBlock <DadTuple>(); Kernel = kernel; DownstreamOperators = downstreamOperators; TupleIdGenerator = tupleIdGenerator; Group = group; outputFilePath = GetOutputFileName(); }
private void Process(DadTuple inputTuple) { Console.WriteLine("[CONSUMER] Starting computation of {0}.", inputTuple); // Process the tuple var computedTuples = Kernel.execute(inputTuple.Content); if (computedTuples.Count == 0) { return; } var outTuples = computedTuples.Select(t => new DadTuple(TupleIdGenerator.NextTupleId(), t)).ToList(); Console.WriteLine("[CONSUMER] Sharing results of computation of {0} to group.", inputTuple); Group.RMSend(p => p.SaveProcessedTuples(inputTuple.Id, outTuples)); //Group.LocalReplica.SaveProcessedTuples(inputTuple.Id, outTuples); // Is the last operator in the acyclic graph if (DownstreamOperators.Count == 0) { // Output to a file (if we are the leader) OutputToFile(outTuples); } else { // Send the tuples SendOutputTuples(outTuples); } // Log LogToPuppetMaster(outTuples); Group.RMSend(p => p.DeliveredTuples(inputTuple.Id, outTuples.Select(ot => ot.Id).ToList())); }
public static void Main(string[] args) { // Check args validity if (args.Length != 3) { Console.WriteLine("[Operator] Usage: OperatorExecutable <serialized_operator> <replicaID>!"); Console.WriteLine("[Operator] Received: "); foreach (string s in args) { Console.WriteLine(s); } exiting(5); return; } // Parse operator string serializedOperator = args[0]; Operator operatorData; try { operatorData = Operator.deserialize(serializedOperator); } catch (Exception e) { Console.WriteLine("[Operator] Couldn't deserialize the operator. Cause: {0}", e.Message); Console.WriteLine("[Operator] Received: {0}", serializedOperator); exiting(10); return; } // Parse replicaId int replicaID; try { replicaID = int.Parse(args[1]); if (replicaID >= operatorData.replicaURLs.Count) { throw new Exception(); } } catch (Exception) { Console.WriteLine("[REPLICA] ReplicaID was not a valid or out of range integer!"); exiting(5); return; } myURL = operatorData.replicaURLs[replicaID]; Console.WriteLine("------------------------------------------------------------------"); Console.WriteLine("[REPLICA] Operator: {0}({1})! URL: {2}", operatorData.id, replicaID, myURL); Console.WriteLine("------------------------------------------------------------------"); // initializeOperatorSpec var kernel = initializeOperatorSpec(operatorData); var downstreamOperators = SetUpDownstream(operatorData); // Connect to the puppet master string puppetMasterLoggerUrl = args[2]; try { PuppetMaster = (IPuppetMaster)Activator.GetObject(typeof(IPuppetMaster), puppetMasterLoggerUrl); Console.WriteLine("[Replica] Created proxy for puppetMaster at {0}", puppetMasterLoggerUrl); PuppetMaster.log("Replica " + operatorData.id + " of operator " + myURL + " launched."); } catch (Exception e) { Console.WriteLine("[Operator] Unable to talk to the puppetMaster at: {0}. Cause: {1}", puppetMasterLoggerUrl, e.Message); } //System.Diagnostics.Debugger.Launch(); var localRoutingStrategy = RoutingPolicyToStrategy(operatorData.routingPolicy, operatorData.HashingField); var localReplica = new LocalReplica(operatorData.id, replicaID, null); var tupleIdGenerator = new TupleIdGenerator(operatorData.id, replicaID); var monitoringProcess = new MonitoringProcess(); var group = new GroupManager(SetUpGroup(operatorData, replicaID, localReplica), localReplica, monitoringProcess, localRoutingStrategy); var tupleProcessor = new TupleProcessor(kernel, downstreamOperators, tupleIdGenerator, group); thisReplica = new ReplicaServices(tupleProcessor, group, monitoringProcess, operatorData, operatorData.id, replicaID); localReplica.SetProxy(thisReplica); // Publish the service string replicaURL = operatorData.replicaURLs[replicaID]; string serviceName = getNameFromURL(replicaURL); try { int servicePort = getPortFromURL(replicaURL); TcpChannel channel = new TcpChannel(servicePort); // @SEE: I think because the PCS has already registered a channel and this processe is a child of it we can´t register again? //ChannelServices.RegisterChannel(channel, false); // @SEE: should be true?? Type serviceType = typeof(ReplicaServices); RemotingServices.Marshal(thisReplica, serviceName, serviceType); System.Console.WriteLine("Started Operator with name {0} at port {1}.", serviceName, servicePort); } catch (Exception e) { Console.WriteLine("[Operator] Unable to publish {0}. Cause: {1}", replicaURL, e.Message); exiting(20); return; } // See if we need to read from a file foreach (string fileName in operatorData.inputFiles) { readTuplesFromFile(INPUT_FOLDER_PATH + fileName, tupleIdGenerator); } Console.ReadLine(); }