Exemplo n.º 1
0
        /* Parse given input file into tupleBuffer */
        private static void readTuplesFromFile(string filepath, TupleIdGenerator tupleIdGenerator)
        {
            TextReader freader;
            string     line;

            try {
                freader = File.OpenText(filepath);
            } catch (Exception e) {
                Console.WriteLine("[ERROR] Unable to Open the file: \"{0}\". Proceding without it. {1}", filepath, e.Message);
                return;
            }

            Console.WriteLine("[Operator] Reading from file: {0}", filepath);
            while ((line = freader.ReadLine()) != null)
            {
                if (line.StartsWith("%")) // ignore comments
                {
                    continue;
                }

                var tuple = line.Split(',').ToList();
                for (int i = 0; i < tuple.Count; i++)
                {
                    tuple[i] = tuple[i].Trim(' '); // strip whitespaces
                }
                thisReplica.Flow(new DadTuple(tupleIdGenerator.NextTupleId(), tuple));
            }
        }
Exemplo n.º 2
0
        public TupleProcessor(AbstractKernel kernel, List <DownstreamOperator> downstreamOperators, TupleIdGenerator tupleIdGenerator, GroupManager group)
        {
            TupleBuffer         = new BufferBlock <DadTuple>();
            Kernel              = kernel;
            DownstreamOperators = downstreamOperators;
            TupleIdGenerator    = tupleIdGenerator;
            Group = group;

            outputFilePath = GetOutputFileName();
        }
Exemplo n.º 3
0
        private void Process(DadTuple inputTuple)
        {
            Console.WriteLine("[CONSUMER] Starting computation of {0}.", inputTuple);

            // Process the tuple
            var computedTuples = Kernel.execute(inputTuple.Content);

            if (computedTuples.Count == 0)
            {
                return;
            }

            var outTuples = computedTuples.Select(t => new DadTuple(TupleIdGenerator.NextTupleId(), t)).ToList();

            Console.WriteLine("[CONSUMER] Sharing results of computation of {0} to group.", inputTuple);
            Group.RMSend(p => p.SaveProcessedTuples(inputTuple.Id, outTuples));
            //Group.LocalReplica.SaveProcessedTuples(inputTuple.Id, outTuples);


            // Is the last operator in the acyclic graph
            if (DownstreamOperators.Count == 0)
            {
                // Output to a file (if we are the leader)
                OutputToFile(outTuples);
            }
            else
            {
                // Send the tuples
                SendOutputTuples(outTuples);
            }

            // Log
            LogToPuppetMaster(outTuples);

            Group.RMSend(p => p.DeliveredTuples(inputTuple.Id, outTuples.Select(ot => ot.Id).ToList()));
        }
Exemplo n.º 4
0
        public static void Main(string[] args)
        {
            // Check args validity
            if (args.Length != 3)
            {
                Console.WriteLine("[Operator] Usage: OperatorExecutable <serialized_operator> <replicaID>!");
                Console.WriteLine("[Operator] Received: ");
                foreach (string s in args)
                {
                    Console.WriteLine(s);
                }
                exiting(5);
                return;
            }

            // Parse operator
            string   serializedOperator = args[0];
            Operator operatorData;

            try {
                operatorData = Operator.deserialize(serializedOperator);
            } catch (Exception e) {
                Console.WriteLine("[Operator] Couldn't deserialize the operator. Cause: {0}", e.Message);
                Console.WriteLine("[Operator] Received: {0}", serializedOperator);
                exiting(10);
                return;
            }

            // Parse replicaId
            int replicaID;

            try {
                replicaID = int.Parse(args[1]);
                if (replicaID >= operatorData.replicaURLs.Count)
                {
                    throw new Exception();
                }
            } catch (Exception) {
                Console.WriteLine("[REPLICA] ReplicaID was not a valid or out of range integer!");
                exiting(5);
                return;
            }
            myURL = operatorData.replicaURLs[replicaID];

            Console.WriteLine("------------------------------------------------------------------");
            Console.WriteLine("[REPLICA] Operator: {0}({1})! URL: {2}", operatorData.id, replicaID, myURL);
            Console.WriteLine("------------------------------------------------------------------");


            // initializeOperatorSpec
            var kernel = initializeOperatorSpec(operatorData);
            var downstreamOperators = SetUpDownstream(operatorData);

            // Connect to the puppet master
            string puppetMasterLoggerUrl = args[2];

            try {
                PuppetMaster = (IPuppetMaster)Activator.GetObject(typeof(IPuppetMaster), puppetMasterLoggerUrl);
                Console.WriteLine("[Replica] Created proxy for puppetMaster at {0}", puppetMasterLoggerUrl);

                PuppetMaster.log("Replica " + operatorData.id + " of operator " + myURL + " launched.");
            } catch (Exception e) {
                Console.WriteLine("[Operator] Unable to talk to the puppetMaster at: {0}. Cause: {1}", puppetMasterLoggerUrl, e.Message);
            }

            //System.Diagnostics.Debugger.Launch();
            var localRoutingStrategy = RoutingPolicyToStrategy(operatorData.routingPolicy, operatorData.HashingField);
            var localReplica         = new LocalReplica(operatorData.id, replicaID, null);
            var tupleIdGenerator     = new TupleIdGenerator(operatorData.id, replicaID);
            var monitoringProcess    = new MonitoringProcess();
            var group          = new GroupManager(SetUpGroup(operatorData, replicaID, localReplica), localReplica, monitoringProcess, localRoutingStrategy);
            var tupleProcessor = new TupleProcessor(kernel, downstreamOperators, tupleIdGenerator, group);

            thisReplica = new ReplicaServices(tupleProcessor, group, monitoringProcess, operatorData, operatorData.id, replicaID);
            localReplica.SetProxy(thisReplica);

            // Publish the service
            string replicaURL  = operatorData.replicaURLs[replicaID];
            string serviceName = getNameFromURL(replicaURL);

            try {
                int servicePort = getPortFromURL(replicaURL);

                TcpChannel channel = new TcpChannel(servicePort);
                // @SEE: I think because the PCS has already registered a channel and this processe is a child of it we can´t register again?
                //ChannelServices.RegisterChannel(channel, false); // @SEE: should be true??

                Type serviceType = typeof(ReplicaServices);
                RemotingServices.Marshal(thisReplica, serviceName, serviceType);

                System.Console.WriteLine("Started Operator with name {0} at port {1}.", serviceName, servicePort);
            }
            catch (Exception e) {
                Console.WriteLine("[Operator] Unable to publish {0}. Cause: {1}", replicaURL, e.Message);
                exiting(20);
                return;
            }

            // See if we need to read from a file
            foreach (string fileName in operatorData.inputFiles)
            {
                readTuplesFromFile(INPUT_FOLDER_PATH + fileName, tupleIdGenerator);
            }

            Console.ReadLine();
        }