Example #1
0
        //public int receivePort, sendPort;

        /*
         * public Slave(SlaveArgJson arg){
         *  this.id = arg.id;
         *  this.ip = Program.stringToIP(this.ip_string = arg.adress);
         *  this.coInitializer();
         * }*/
        public static List <Slave> staticInit()
        {
            //todo: in hotRestart chiama esplicitamente tutti gli staticInit e aggiungi clausole per chiudere i client già aperti sovrascrivendoli.
            //cleanup for HotRestart
            if (Slave.senderToSlave != null)
            {
                senderToSlave.Close();
            }
            if (Slave.ensureUniqueID != null)
            {
                ensureUniqueID.Clear();
                lock (Slave.all) foreach (Slave s in Slave.all)
                    {
                        Slave.Remove(s);
                    }
            }
            //real initialization
            Slave.senderToSlave = new UdpClient()
            {
                EnableBroadcast = true
            };
            Slave.senderToSlave.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.NoDelay, true);//nagle's algorithm should work only in tpc, but i want to be sure that it will not execute, that would be a problem.
            Slave.senderToSlave.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.NoChecksum, false);
            Slave.senderToSlave.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
            Slave.senderToSlave.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, false);
            Slave.senderToSlave.ExclusiveAddressUse = false;
            Slave.senderToSlave.EnableBroadcast     = true;
            Slave.all            = new List <Slave>();
            Slave.ensureUniqueID = new Dictionary <ulong, Slave>();
            return(Slave.all);
        }
Example #2
0
 public Slave coInitializer()
 {
     lock (Slave.all) {
         if (Slave.alreadyRegistered(this))
         {
             return(Slave.getFromID(this.id));
         }
         Slave.all.Add(this);
         if (Slave.ensureUniqueID.ContainsKey(this.id))
         {
             Program.pex("Argument ERROR: found 2 slaves with same id ( " + this.id + " ).");
         }
         else
         {
             Slave.ensureUniqueID.Add(this.id, this);
         }
     }
     if (this.isSelf)
     {
         if (Slave.self != null)
         {
             Program.pex("Argument ERROR: only one Slave must be marked as \"self\" ");
         }
         Slave.self = this;
         if (Program.args.dinamicallyStarted)
         {
             DinamicallyStart();
         }
     }
     return(this);
 }
Example #3
0
        public static Slave findNextMaster()
        {
            Slave[] masterPrioList = Slave.sortByMasterPrio();
            // if it is dinamically executed and there is no known master,
            // but there are peer colleagues means one of them is the current master even if this slave might have higher master priority than current master.
            // in this case it will not replace the current master, because it's useless while it is working and because this slave have a empty queue.ù
            // so a dinamically started slave can be the master only if it is the only one.

            if (Master.currentMaster == null)
            {
                if (Program.args.dinamicallyStarted)   //todo: uno slave apena partito potrebbe non conoscere il master finchè non riceve la reply messagetype.SlaveList
                {
                    if (masterPrioList[0] == Slave.self && masterPrioList.Length > 1)
                    {
                        return(masterPrioList[1]);
                    }
                    else
                    {
                        return(masterPrioList[0]);
                    }
                }
                else
                {
                    return(masterPrioList[0]);
                }
            }
            // guaranteed to have at least 2 elements: the previous master (now crashed) and the executing slave program.
            // except during initialitization when there can be 1 without triggering error. (the master = this program)
            Slave nextmaster = masterPrioList[0] == Master.currentMaster ? masterPrioList[0] : masterPrioList[1];

            return(nextmaster);
        }
Example #4
0
        public int CompareTo(object S)
        {
            if (!(S is Slave))
            {
                Program.pe("Wrong comparison usage: Slave with " + S.GetType().ToString() + ";"); return(-1);
            }
            Slave s = (Slave)S;

            return(this.id > s.id ? 1 : (this.id == s.id ? 0 : -1));
        }
Example #5
0
        public static Slave deserializeOrGet(string str)
        {
            Slave tmp = null;

            try { tmp = ((Slave)JsonConvert.DeserializeObject(str)).coInitializer(); }
            catch (Exception e) { Program.pe("Failed to deserialize Slave object: " + str, e); }
            if (tmp == null)
            {
                return(null);
            }
            return(tmp);
        }
Example #6
0
        public static string[] fakeinput()
        {
            string         desktop   = Environment.GetFolderPath(System.Environment.SpecialFolder.DesktopDirectory);
            bool           exclBind  = false;
            string         kafkaHost = "localhost";//"192.168.1.8";//"localhost"
            StartupArgJson json      = new StartupArgJson()
            {
                myPartitionNumber      = 0,
                partitionNumbers_Total = 2,
                broadcastPort_Tool     = 20001,
                toolReceiverThreads    = 1, //exclBind ? 1 : (int)Math.Ceiling(Environment.ProcessorCount / 2.0),//it's logical, not physical! it is the maximum number of thread executable simultaneously.
                slaveReceiverThreads   = 1, // exclBind ? 1 : (int)Math.Floor(Environment.ProcessorCount / 2.0),
                enableGUI           = true,
                enablePrintSlave    = false,
                enablePrintTool     = false,
                broadcastAddress    = "192.168.1.255",
                criticalErrFile     = desktop + @"\Listener_CriticalErrors.txt",
                errFile             = desktop + @"\Listener_Errors.txt",
                logFile             = desktop + @"\Listener_EventLog.txt",
                toolMsgFile         = null, //desktop + @"\Listener_ToolLog.txt",
                slaveMsgFile        = desktop + @"\Listener_SlaveLog.txt",
                logToolMsgOnReceive = false,
                exclusiveBind       = exclBind,
                KafkaNodes          = "http://" + kafkaHost + ":9093, http://" + kafkaHost + ":9094, http://" + kafkaHost + ":9095",
                KafkaTopic          = "toolsEvents",
                benchmark           = true,
            };

            json.slaveNotifyMode_Batch = 100;
            json.dinamicallyStarted    = false;
            json.broadcastPort_Slaves  = 20002 + json.myPartitionNumber;


            ulong guid = Program.GetMACAddress();
            int   replicationDegree = 3;

            json.replicatorsList = new List <Slave>(replicationDegree);

            for (int i = 0; i < replicationDegree; i++)
            {
                Slave replica = new Slave();
                replica.ip_string = "192.168.1." + (100 + i);
                replica.id        = guid + (ulong)i;
                replica.isSelf    = i == 0;
                json.replicatorsList.Add(replica);
            }
            string s = json.ToString();

            return(new string[] { json.ToString() });
        }
        public void launchToOutput()
        {
            switch (this.type)
            {
            case MessageType.provideSlaveList:
            case MessageType.masterChange:
            case MessageType.dinamicallyAddSlave:
            case MessageType.dinamicallyRemoveSlave:
            case MessageType.confirmMessageSuccess_Batch:
            case MessageType.confirmMessageSuccess_Single:
                Slave.sendToAll(this); break;

            case MessageType.xml:
            case MessageType.uninitialized:
            default: Program.pe("launchToOutput should never be called on this message type: " + this.type); return;
            }
        }
Example #8
0
 public static void masterCrashCheckLoop0()
 {
     try {
         while (true)
         {
             if (masterIsCrashedCheck())
             {
                 //what if a dinamically added slave that is not acknowledged by all becomes the master? devo passare direttamente il master scelto per forza.
                 //Slave.sendToAll(new Message(MessageType.masterCrashed, Master.currentMaster.serialize()));
                 Slave newmaster = findNextMaster();
                 new myMessage(MessageType.masterChange, newmaster.serialize()).launchToOutput();
                 Master.changeMaster(newmaster);//master checker loop
             }
             Thread.Sleep(Master.crashCheckInterval);
         }
     } catch (ThreadAbortException ex) { Program.p("Stopped thread MasterCrashCheckLoop: ", ex); return; }
 }
Example #9
0
        public Master(Slave s = null)
        {
            Master.master = this;
            Slave        selfDebug = Slave.self;
            List <Slave> alldebug  = Slave.all;

            if (s == null)
            {
                s = findNextMaster();
            }
            changeMaster(s);//init
            if (iAmTheMaster || MasterCrashChecker != null)
            {
                return;
            }
            MasterCrashChecker      = new Thread(masterCrashCheckLoop);
            MasterCrashChecker.Name = "Master crash checker (Slave only)";
            MasterCrashChecker.Start();
        }
Example #10
0
 public static void changeMaster(Slave s)
 {
     if (Master.currentMaster == s)
     {
         return;
     }
     if (Slave.self == s)   //if i became the master...
     {
         if (MasterCrashChecker != null)
         {
             MasterCrashChecker.Abort(); MasterCrashChecker = null;
         }                                                                                         //todo: sempre setta a null i thread terminati.
         Master.currentMaster = s;
         Master.becomeMaster();
     }
     else
     {
         Master.currentMaster = s;
     }
 }
Example #11
0
        /// <summary>
        /// This function allows to change parameters at run-time without ceasing to receive broadcasted message.
        /// To trigger this function, a broadcast message must be sent to the arguments.broadcastPort_Slaves port,
        /// containing serialized object instance of myMessage class with msg.key = MessageType.argumentChange and msg.data = Json-stringified-arguments.
        /// </summary>
        public static void HotRestart()
        {
            IPAddress tmp = IPAddress.Parse(Program.args.broadcastAddress);

            Slave.broadcastToSlaveEP = new IPEndPoint(tmp, Program.args.broadcastPort_Slaves);
            Master.canPublish        = new Semaphore(0, int.MaxValue);//todo: check
            //myKafka.kafkatest();
            //myKafka.send(new myMessage[]{new myMessage(MessageType.confirmMessageSuccess_Single, "ProducerA first test")});
            //return;

            ReceiverTool.staticInit();
            SlaveReceiver.staticInit();
            Slave.staticInit();
            myKafka.staticInit();
            SlaveMsgConsumer.staticInit();

            /*var Client = new UdpClient();
             * var RequestData = Encoding.ASCII.GetBytes("SomeRequestData");
             * var ServerEp = new IPEndPoint(IPAddress.Any, 0);
             *
             * Client.EnableBroadcast = true;
             * Client.Send(RequestData, RequestData.Length, new IPEndPoint(IPAddress.Broadcast, 8888));
             */
            //Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
            //sock.Poll(1,SelectMode.SelectWrite);
            //tmp = IPAddress.Broadcast;
            //Slave.senderToSlave.Client.Bind(Program.broadcastEP);
            foreach (Slave a in args.replicatorsList)
            {
                a.coInitializer();
            }
            if (Slave.self == null)
            {
                pex("Argument error: Exactly one slave must have \"isSelf = true\". It is needed to know what slave id this instance has.");
            }
            new Master();
            while (ReceiverTool.all.Count < args.toolReceiverThreads)
            {
                new ReceiverTool();
            }
            while (SlaveReceiver.all.Count < args.slaveReceiverThreads)
            {
                new SlaveReceiver();
            }
            //only happening in case of hot restart for argument change: the receiver threads will never stop collecting.
            //NB: in caso di riduzione dei thread potrebbe verificarsi la perdita di un messaggio se il thread viene interrotto tra la ricezione e l'accodamento. fare un fermo sincronizzato però implicherebbe delle operazioni durante la normale esecuzione che rallenterebbero il throughput.
            while (ReceiverTool.all.Count > args.toolReceiverThreads)
            {
                ReceiverTool.RemoveOne();
            }
            while (SlaveReceiver.all.Count > args.slaveReceiverThreads)
            {
                SlaveReceiver.RemoveOne();
            }
            for (int i = 0; i < ReceiverTool.all.Count; i++)
            {
                ReceiverTool.all[i].myThread.Name = "Tool Broadcast Broadcast Receiver " + (i + 1) + "/" + ReceiverTool.all.Count;
            }
            for (int i = 0; i < SlaveReceiver.all.Count; i++)
            {
                SlaveReceiver.all[i].myThread.Name = "IntraCommunication Receiver " + (i + 1) + "/" + SlaveReceiver.all.Count;
            }
            new SlaveMsgConsumer().start();
            SlaveReceiver.Start();
            ReceiverTool.Start();
            //MessageBox.Show("Done");
            if (args.enableGUI && GUI.thiss == null)
            {
                //args.enableGUI = false;//debug only
                Thread.CurrentThread.Name = "MainGuiThread";
                Application.EnableVisualStyles();
                Application.SetCompatibleTextRenderingDefault(false);
                Application.Run(new GUI());
            }
        }
Example #12
0
 public static void Remove(Slave s)
 {
     lock (Slave.all) { Slave.all.Remove(s); Slave.ensureUniqueID.Remove(s.id); }
 }
Example #13
0
 public static bool alreadyRegistered(Slave s)
 {
     return(null != getFromID(s.id));
 }
        /// <summary>
        /// processing executed after message got dequeued.
        /// </summary>
        public void consume()
        {
            Program.logSlave("consuming :" + this.ToPrintString());
            Slave s;
            ulong id;
            int   removedCount;

            switch (type)
            {
            case MessageType.argumentChange:
                StartupArgJson args;
                try { args = StartupArgJson.deserialize(this.data); } catch (Exception e) {
                    Program.pe(this.type + " body is not deserializable: " + this.data, e); break;
                }
                if (args == null || !args.Validate())
                {
                    Program.pe(this.type + " body is deserializable but with invalid content: " + this.data); break;
                }
                Program.args = args;
                Master.MasterCrashChecker.Abort();
                Master.MasterCrashChecker = null;
                new Thread(Program.HotRestart).Start();

                /*
                 * string[] arr = this.data.Split(myMessage.separator);
                 * foreach (string str in arr) {
                 *  string[] kv = str.Split(myMessage.secondSeparator);
                 *  ulong slaveID;
                 *  if (!ulong.TryParse(kv[0], out slaveID)) { Program.pe("Unexpected slaveID key ("+kv[0]+") found in the body of messagetype."+this.type); continue; }
                 *
                 * }*/
                //todo: crea anche un software che generi messaggi di ripartizionamento per gestire dinamicamente tutte le partizioni, un supermaster
                break;

            case MessageType.masterChange:
                //if required in future trigger messageType.dinamicallyAddSlave, per ora va tutto bene anche se il nuovo master non era nella lista slaves.
                string[] split = this.data.Split(myMessage.separator);
                if (!ulong.TryParse(split[0], out id))
                {
                    Program.pe(this.type + " have non-numerical body; expected two numeric id separated by a '" + myMessage.separator + "', found instead: " + this.data); break;
                }
                Slave oldMaster = Slave.getFromID(id);
                if (oldMaster == null)
                {
                    break;
                }
                if (!ulong.TryParse(split[0], out id))
                {
                    Program.pe(this.type + " have non-numerical body; expected two numeric id separated by a '" + myMessage.separator + "', found instead: " + this.data); break;
                }
                Slave newMaster = Slave.getFromID(id);
                if (newMaster == null)
                {
                    break;
                }
                if (Master.currentMaster == oldMaster)
                {
                    Master.changeMaster(newMaster);                                       //master checker msg received
                }
                Slave.Remove(oldMaster);
                break;

            case MessageType.dinamicallyRemoveSlave:
                if (!ulong.TryParse(this.data, out id))
                {
                    Program.pe(this.type + " have non-numerical body; expected numeric id, found: " + this.data); break;
                }
                s = Slave.getFromID(id);
                if (s == null)
                {
                    break;
                }
                Slave.Remove(s);
                break;

            case MessageType.dinamicallyAddSlave:
                Slave.deserializeOrGet(this.data);
                if (Master.iAmTheMaster)
                {
                    myMessage m = new myMessage(MessageType.provideSlaveList, "");
                    foreach (Slave s2 in Slave.all)
                    {
                        m.data += ";" + s2.serialize();
                    }
                    m.data = m.data.Substring(1);
                    m.launchToOutput();
                }
                break;

            case MessageType.provideSlaveList:
                Volatile.Write(ref Master.lastMasterUpdate, DateTime.Now.Ticks);
                string[] jsons = this.data.Split(myMessage.separator);
                lock (Slave.all) foreach (string str in jsons)
                    {
                        Slave.deserializeOrGet(str);
                    }
                break;

            case MessageType.confirmMessageSuccess_Single:
                Volatile.Write(ref Master.lastMasterUpdate, DateTime.Now.Ticks);
                removedCount = ReceiverTool.messageQueue.get(this.data, true) == null ? 0 : 1;
                Program.logSlave(removedCount + " removed from queue.");
                break;

            case MessageType.confirmMessageSuccess_Batch:
                Volatile.Write(ref Master.lastMasterUpdate, DateTime.Now.Ticks);
                removedCount = ReceiverTool.messageQueue.getOlderThan(this.data, true).Count;
                Program.logSlave(removedCount + " removed from queue.");
                break;

            case MessageType.xml: Program.pe("xml messages should be handled in Master thread without consuming."); return;

            default:
            case MessageType.uninitialized: Program.pe("uninitialized message consumed"); return;
            }
        }