//public int receivePort, sendPort; /* * public Slave(SlaveArgJson arg){ * this.id = arg.id; * this.ip = Program.stringToIP(this.ip_string = arg.adress); * this.coInitializer(); * }*/ public static List <Slave> staticInit() { //todo: in hotRestart chiama esplicitamente tutti gli staticInit e aggiungi clausole per chiudere i client già aperti sovrascrivendoli. //cleanup for HotRestart if (Slave.senderToSlave != null) { senderToSlave.Close(); } if (Slave.ensureUniqueID != null) { ensureUniqueID.Clear(); lock (Slave.all) foreach (Slave s in Slave.all) { Slave.Remove(s); } } //real initialization Slave.senderToSlave = new UdpClient() { EnableBroadcast = true }; Slave.senderToSlave.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.NoDelay, true);//nagle's algorithm should work only in tpc, but i want to be sure that it will not execute, that would be a problem. Slave.senderToSlave.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.NoChecksum, false); Slave.senderToSlave.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); Slave.senderToSlave.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, false); Slave.senderToSlave.ExclusiveAddressUse = false; Slave.senderToSlave.EnableBroadcast = true; Slave.all = new List <Slave>(); Slave.ensureUniqueID = new Dictionary <ulong, Slave>(); return(Slave.all); }
public Slave coInitializer() { lock (Slave.all) { if (Slave.alreadyRegistered(this)) { return(Slave.getFromID(this.id)); } Slave.all.Add(this); if (Slave.ensureUniqueID.ContainsKey(this.id)) { Program.pex("Argument ERROR: found 2 slaves with same id ( " + this.id + " )."); } else { Slave.ensureUniqueID.Add(this.id, this); } } if (this.isSelf) { if (Slave.self != null) { Program.pex("Argument ERROR: only one Slave must be marked as \"self\" "); } Slave.self = this; if (Program.args.dinamicallyStarted) { DinamicallyStart(); } } return(this); }
public static Slave findNextMaster() { Slave[] masterPrioList = Slave.sortByMasterPrio(); // if it is dinamically executed and there is no known master, // but there are peer colleagues means one of them is the current master even if this slave might have higher master priority than current master. // in this case it will not replace the current master, because it's useless while it is working and because this slave have a empty queue.ù // so a dinamically started slave can be the master only if it is the only one. if (Master.currentMaster == null) { if (Program.args.dinamicallyStarted) //todo: uno slave apena partito potrebbe non conoscere il master finchè non riceve la reply messagetype.SlaveList { if (masterPrioList[0] == Slave.self && masterPrioList.Length > 1) { return(masterPrioList[1]); } else { return(masterPrioList[0]); } } else { return(masterPrioList[0]); } } // guaranteed to have at least 2 elements: the previous master (now crashed) and the executing slave program. // except during initialitization when there can be 1 without triggering error. (the master = this program) Slave nextmaster = masterPrioList[0] == Master.currentMaster ? masterPrioList[0] : masterPrioList[1]; return(nextmaster); }
public int CompareTo(object S) { if (!(S is Slave)) { Program.pe("Wrong comparison usage: Slave with " + S.GetType().ToString() + ";"); return(-1); } Slave s = (Slave)S; return(this.id > s.id ? 1 : (this.id == s.id ? 0 : -1)); }
public static Slave deserializeOrGet(string str) { Slave tmp = null; try { tmp = ((Slave)JsonConvert.DeserializeObject(str)).coInitializer(); } catch (Exception e) { Program.pe("Failed to deserialize Slave object: " + str, e); } if (tmp == null) { return(null); } return(tmp); }
public static string[] fakeinput() { string desktop = Environment.GetFolderPath(System.Environment.SpecialFolder.DesktopDirectory); bool exclBind = false; string kafkaHost = "localhost";//"192.168.1.8";//"localhost" StartupArgJson json = new StartupArgJson() { myPartitionNumber = 0, partitionNumbers_Total = 2, broadcastPort_Tool = 20001, toolReceiverThreads = 1, //exclBind ? 1 : (int)Math.Ceiling(Environment.ProcessorCount / 2.0),//it's logical, not physical! it is the maximum number of thread executable simultaneously. slaveReceiverThreads = 1, // exclBind ? 1 : (int)Math.Floor(Environment.ProcessorCount / 2.0), enableGUI = true, enablePrintSlave = false, enablePrintTool = false, broadcastAddress = "192.168.1.255", criticalErrFile = desktop + @"\Listener_CriticalErrors.txt", errFile = desktop + @"\Listener_Errors.txt", logFile = desktop + @"\Listener_EventLog.txt", toolMsgFile = null, //desktop + @"\Listener_ToolLog.txt", slaveMsgFile = desktop + @"\Listener_SlaveLog.txt", logToolMsgOnReceive = false, exclusiveBind = exclBind, KafkaNodes = "http://" + kafkaHost + ":9093, http://" + kafkaHost + ":9094, http://" + kafkaHost + ":9095", KafkaTopic = "toolsEvents", benchmark = true, }; json.slaveNotifyMode_Batch = 100; json.dinamicallyStarted = false; json.broadcastPort_Slaves = 20002 + json.myPartitionNumber; ulong guid = Program.GetMACAddress(); int replicationDegree = 3; json.replicatorsList = new List <Slave>(replicationDegree); for (int i = 0; i < replicationDegree; i++) { Slave replica = new Slave(); replica.ip_string = "192.168.1." + (100 + i); replica.id = guid + (ulong)i; replica.isSelf = i == 0; json.replicatorsList.Add(replica); } string s = json.ToString(); return(new string[] { json.ToString() }); }
public void launchToOutput() { switch (this.type) { case MessageType.provideSlaveList: case MessageType.masterChange: case MessageType.dinamicallyAddSlave: case MessageType.dinamicallyRemoveSlave: case MessageType.confirmMessageSuccess_Batch: case MessageType.confirmMessageSuccess_Single: Slave.sendToAll(this); break; case MessageType.xml: case MessageType.uninitialized: default: Program.pe("launchToOutput should never be called on this message type: " + this.type); return; } }
public static void masterCrashCheckLoop0() { try { while (true) { if (masterIsCrashedCheck()) { //what if a dinamically added slave that is not acknowledged by all becomes the master? devo passare direttamente il master scelto per forza. //Slave.sendToAll(new Message(MessageType.masterCrashed, Master.currentMaster.serialize())); Slave newmaster = findNextMaster(); new myMessage(MessageType.masterChange, newmaster.serialize()).launchToOutput(); Master.changeMaster(newmaster);//master checker loop } Thread.Sleep(Master.crashCheckInterval); } } catch (ThreadAbortException ex) { Program.p("Stopped thread MasterCrashCheckLoop: ", ex); return; } }
public Master(Slave s = null) { Master.master = this; Slave selfDebug = Slave.self; List <Slave> alldebug = Slave.all; if (s == null) { s = findNextMaster(); } changeMaster(s);//init if (iAmTheMaster || MasterCrashChecker != null) { return; } MasterCrashChecker = new Thread(masterCrashCheckLoop); MasterCrashChecker.Name = "Master crash checker (Slave only)"; MasterCrashChecker.Start(); }
public static void changeMaster(Slave s) { if (Master.currentMaster == s) { return; } if (Slave.self == s) //if i became the master... { if (MasterCrashChecker != null) { MasterCrashChecker.Abort(); MasterCrashChecker = null; } //todo: sempre setta a null i thread terminati. Master.currentMaster = s; Master.becomeMaster(); } else { Master.currentMaster = s; } }
/// <summary> /// This function allows to change parameters at run-time without ceasing to receive broadcasted message. /// To trigger this function, a broadcast message must be sent to the arguments.broadcastPort_Slaves port, /// containing serialized object instance of myMessage class with msg.key = MessageType.argumentChange and msg.data = Json-stringified-arguments. /// </summary> public static void HotRestart() { IPAddress tmp = IPAddress.Parse(Program.args.broadcastAddress); Slave.broadcastToSlaveEP = new IPEndPoint(tmp, Program.args.broadcastPort_Slaves); Master.canPublish = new Semaphore(0, int.MaxValue);//todo: check //myKafka.kafkatest(); //myKafka.send(new myMessage[]{new myMessage(MessageType.confirmMessageSuccess_Single, "ProducerA first test")}); //return; ReceiverTool.staticInit(); SlaveReceiver.staticInit(); Slave.staticInit(); myKafka.staticInit(); SlaveMsgConsumer.staticInit(); /*var Client = new UdpClient(); * var RequestData = Encoding.ASCII.GetBytes("SomeRequestData"); * var ServerEp = new IPEndPoint(IPAddress.Any, 0); * * Client.EnableBroadcast = true; * Client.Send(RequestData, RequestData.Length, new IPEndPoint(IPAddress.Broadcast, 8888)); */ //Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); //sock.Poll(1,SelectMode.SelectWrite); //tmp = IPAddress.Broadcast; //Slave.senderToSlave.Client.Bind(Program.broadcastEP); foreach (Slave a in args.replicatorsList) { a.coInitializer(); } if (Slave.self == null) { pex("Argument error: Exactly one slave must have \"isSelf = true\". It is needed to know what slave id this instance has."); } new Master(); while (ReceiverTool.all.Count < args.toolReceiverThreads) { new ReceiverTool(); } while (SlaveReceiver.all.Count < args.slaveReceiverThreads) { new SlaveReceiver(); } //only happening in case of hot restart for argument change: the receiver threads will never stop collecting. //NB: in caso di riduzione dei thread potrebbe verificarsi la perdita di un messaggio se il thread viene interrotto tra la ricezione e l'accodamento. fare un fermo sincronizzato però implicherebbe delle operazioni durante la normale esecuzione che rallenterebbero il throughput. while (ReceiverTool.all.Count > args.toolReceiverThreads) { ReceiverTool.RemoveOne(); } while (SlaveReceiver.all.Count > args.slaveReceiverThreads) { SlaveReceiver.RemoveOne(); } for (int i = 0; i < ReceiverTool.all.Count; i++) { ReceiverTool.all[i].myThread.Name = "Tool Broadcast Broadcast Receiver " + (i + 1) + "/" + ReceiverTool.all.Count; } for (int i = 0; i < SlaveReceiver.all.Count; i++) { SlaveReceiver.all[i].myThread.Name = "IntraCommunication Receiver " + (i + 1) + "/" + SlaveReceiver.all.Count; } new SlaveMsgConsumer().start(); SlaveReceiver.Start(); ReceiverTool.Start(); //MessageBox.Show("Done"); if (args.enableGUI && GUI.thiss == null) { //args.enableGUI = false;//debug only Thread.CurrentThread.Name = "MainGuiThread"; Application.EnableVisualStyles(); Application.SetCompatibleTextRenderingDefault(false); Application.Run(new GUI()); } }
public static void Remove(Slave s) { lock (Slave.all) { Slave.all.Remove(s); Slave.ensureUniqueID.Remove(s.id); } }
public static bool alreadyRegistered(Slave s) { return(null != getFromID(s.id)); }
/// <summary> /// processing executed after message got dequeued. /// </summary> public void consume() { Program.logSlave("consuming :" + this.ToPrintString()); Slave s; ulong id; int removedCount; switch (type) { case MessageType.argumentChange: StartupArgJson args; try { args = StartupArgJson.deserialize(this.data); } catch (Exception e) { Program.pe(this.type + " body is not deserializable: " + this.data, e); break; } if (args == null || !args.Validate()) { Program.pe(this.type + " body is deserializable but with invalid content: " + this.data); break; } Program.args = args; Master.MasterCrashChecker.Abort(); Master.MasterCrashChecker = null; new Thread(Program.HotRestart).Start(); /* * string[] arr = this.data.Split(myMessage.separator); * foreach (string str in arr) { * string[] kv = str.Split(myMessage.secondSeparator); * ulong slaveID; * if (!ulong.TryParse(kv[0], out slaveID)) { Program.pe("Unexpected slaveID key ("+kv[0]+") found in the body of messagetype."+this.type); continue; } * * }*/ //todo: crea anche un software che generi messaggi di ripartizionamento per gestire dinamicamente tutte le partizioni, un supermaster break; case MessageType.masterChange: //if required in future trigger messageType.dinamicallyAddSlave, per ora va tutto bene anche se il nuovo master non era nella lista slaves. string[] split = this.data.Split(myMessage.separator); if (!ulong.TryParse(split[0], out id)) { Program.pe(this.type + " have non-numerical body; expected two numeric id separated by a '" + myMessage.separator + "', found instead: " + this.data); break; } Slave oldMaster = Slave.getFromID(id); if (oldMaster == null) { break; } if (!ulong.TryParse(split[0], out id)) { Program.pe(this.type + " have non-numerical body; expected two numeric id separated by a '" + myMessage.separator + "', found instead: " + this.data); break; } Slave newMaster = Slave.getFromID(id); if (newMaster == null) { break; } if (Master.currentMaster == oldMaster) { Master.changeMaster(newMaster); //master checker msg received } Slave.Remove(oldMaster); break; case MessageType.dinamicallyRemoveSlave: if (!ulong.TryParse(this.data, out id)) { Program.pe(this.type + " have non-numerical body; expected numeric id, found: " + this.data); break; } s = Slave.getFromID(id); if (s == null) { break; } Slave.Remove(s); break; case MessageType.dinamicallyAddSlave: Slave.deserializeOrGet(this.data); if (Master.iAmTheMaster) { myMessage m = new myMessage(MessageType.provideSlaveList, ""); foreach (Slave s2 in Slave.all) { m.data += ";" + s2.serialize(); } m.data = m.data.Substring(1); m.launchToOutput(); } break; case MessageType.provideSlaveList: Volatile.Write(ref Master.lastMasterUpdate, DateTime.Now.Ticks); string[] jsons = this.data.Split(myMessage.separator); lock (Slave.all) foreach (string str in jsons) { Slave.deserializeOrGet(str); } break; case MessageType.confirmMessageSuccess_Single: Volatile.Write(ref Master.lastMasterUpdate, DateTime.Now.Ticks); removedCount = ReceiverTool.messageQueue.get(this.data, true) == null ? 0 : 1; Program.logSlave(removedCount + " removed from queue."); break; case MessageType.confirmMessageSuccess_Batch: Volatile.Write(ref Master.lastMasterUpdate, DateTime.Now.Ticks); removedCount = ReceiverTool.messageQueue.getOlderThan(this.data, true).Count; Program.logSlave(removedCount + " removed from queue."); break; case MessageType.xml: Program.pe("xml messages should be handled in Master thread without consuming."); return; default: case MessageType.uninitialized: Program.pe("uninitialized message consumed"); return; } }