public QueueRequests(string oldID, ConnectionPack author, IList <IList <string> > result)
{
this.tupleID = oldID;
this.author = author;
this.result = result;
this.backup = true;
}
public void SendCommand(ConnectionPack cp)
{
HashSet <string> set = new HashSet <string>();
foreach (string url in cp.ListUrls)
{
int portIndex = url.IndexOf(':', 4); // skip the first : from the tcp protocol
string pcsUrl = url.Substring(0, portIndex + 1) + "10000/pcs";
set.Add(pcsUrl);
}
foreach (string pcsUrl in set)
{
PuppetMaster.GetInstance().Log("PCS URL: " + pcsUrl);
// Remove comments after pcs implementation
try
{
IPcsSlaveLaunch remoteObj = (IPcsSlaveLaunch)Activator.GetObject(
typeof(IPcsSlaveLaunch), pcsUrl);
remoteObj.Launch(cp);
}
catch (SocketException e)
{
MessageBox.Show("Error connecting to PCS with URL: " + pcsUrl);
PuppetMaster.GetInstance().Log("Error connecting to PCS with URL: " + pcsUrl);
}
}
}
public void receiveReplicaBackup(string oldID, ConnectionPack author, IList <IList <string> > result)
{
try {
remoteProcess.receiveReplicaBackup(oldID, author, result);
} catch (SocketException) {
//Hoppefully someone will notice
}
}
public ClientPuppet(ConnectionPack cp)
{
remotePuppet = (PuppetMasterRemoteServerObject)Activator.GetObject(
typeof(PuppetMasterRemoteServerObject),
"tcp://" + cp.Ip + ":" + cp.Port + "/PuppetMasterRemoteServerObject");
//TODO Exceprion
//if (remotePuppet == null) throw new SocketException();
}
public ClientProcess(ConnectionPack cp)
{
remoteProcess = (ProcessRemoteServerObject)Activator.GetObject(
typeof(ProcessRemoteServerObject),
"tcp://" + cp.Ip + ":" + cp.Port + "/op");
//TODO Exceprion
//if (remoteProcess == null) throw new SocketException();
}
public void reborn(ConnectionPack deadGuy)
{
if (ServerProcess.Instance.Frozen)
{
throw new SocketException();
}
ServerProcess.Instance.reborn(deadGuy);
}
public ClientDaemon(ConnectionPack cp, bool fullLog)
{
remoteDaemon = (DaemonRemoteServerObject)Activator.GetObject(
typeof(DaemonRemoteServerObject),
"tcp://" + cp.Ip + ":" + cp.Port + "/DaemonRemoteServerObject");
remoteDaemon.fullLog(fullLog);
//TODO if (remoteDaemon == null) throw new SocketException();
}
public void receiveReplicaBackup(string oldID, ConnectionPack author, IList <IList <string> > result)
{
if (ServerProcess.Instance.Suiciding)
{
ServerProcess.Instance.addToQueueOfBackup(oldID, author, result);
return;
}
ServerProcess.Instance.receiveReplicaBackup(oldID, author, result);
}
public bool isAlive(ConnectionPack brother)
{
try {
return(remoteProcess.isAlive(brother));
} catch (SocketException) {
//If he is dead he does not count
return(true);
}
}
public override string CreateText(AppObject appobj, ConnectionPack connpack)
{
var va = appobj as VariantVersionAppObject;
if (va == null)
{
return("");
}
return(va.GetVersion().GetAlterSql());
}
private void doOperation(string operation, ConnectionPack cp) {
DADStormProcess.ClientProcess process = new DADStormProcess.ClientProcess(cp);
if(operation.Equals("report", StringComparison.OrdinalIgnoreCase)) {
MemoryStream stream = process.reportBack();
if(stream != null) {
stream.Position = 0;
CSF_metric metric = correctMetric((CSF_metric)js.ReadObject(stream));
metrics.Add(metric);
}
}
if (operation.Equals("reset", StringComparison.OrdinalIgnoreCase)) {
process.reset();
}
}
/// <summary>
/// Method that from a string[] does a command in a single replica
/// </summary>
private void replicaTargetOperations(string[] splitStr)
{
if (splitStr.Length < 3)
{
operatorTargetOperations(splitStr);
return;
}
string command = splitStr[0];
string operator_id = splitStr[1];
//Must be an integer
int process_number = Int32.Parse(splitStr[2]);
List <ConnectionPack> operatorList;
if (operatorsConPacks.TryGetValue(operator_id, out operatorList))
{
ConnectionPack conPack = operatorList [process_number];
DADStormProcess.ClientProcess cprocess = new DADStormProcess.ClientProcess(conPack);
if (command.Equals("freeze", StringComparison.OrdinalIgnoreCase))
{
cprocess.freeze();
logCommand("freezed: " + operator_id + " replica: " + conPack);
}
else if (command.Equals("unfreeze", StringComparison.OrdinalIgnoreCase))
{
cprocess.unfreeze();
logCommand("unfreezed: " + operator_id + " replica: " + conPack);
}
else if (command.Equals("crash", StringComparison.OrdinalIgnoreCase))
{
cprocess.crash();
logCommand("crashed: " + operator_id + " replica: " + conPack);
}
else if (command.Equals("start", StringComparison.OrdinalIgnoreCase))
{
if (firstStart)
{
doFirstStartConnections();
}
cprocess.start();
logCommand("started: " + operator_id + " replica: " + conPack);
}
}
else
{
PuppetError("Operator: " + operator_id + " not in list");
}
}
private void doStatus(ConnectionPack cp)
{
DADStormProcess.ClientProcess process = new DADStormProcess.ClientProcess(cp);
string status = process.status();
if (status.Equals("Machine Failed"))
{
Console.ForegroundColor = ConsoleColor.Red;
}
else
{
Console.ForegroundColor = ConsoleColor.Green;
}
System.Console.WriteLine(cp + " " + status);
Console.ResetColor();
}
public void assignPuppetConPack(ConnectionPack puppetCp)
{
ServerProcess.Instance.PuppetMasterConPack = puppetCp;
}
public void Launch(ConnectionPack input)
{
bool _stateful = false;
bool firstOP = false;
string cmd = input.Cmd;
List <string> urls = input.ListUrls;
List <string> downstreamUrls = input.ReplicaUrlsOutput;
// in order to make a custom import for the first operator
if (input.RoutingTypeToReadFromFile != null)
{
firstOP = true;
}
// split command by keywords
string pattern = @"INPUT OPS|REP FACT|ROUTING|ADDRESS|OPERATOR SPEC";
string[] tokens = Regex.Split(cmd, pattern, RegexOptions.IgnoreCase).Where(s => s != String.Empty).ToArray <string>();
// splitting by 5 keywords should generate 6 tokens
if (tokens.Length != 6)
{
System.Console.WriteLine("Something went wrong while splitting the command!!!");
}
// tokens[0] -> operator name
// tokens[1] -> input file or previous operator
// tokens[2] -> replication factor
// tokens[3] -> routing policy
// tokens[4] -> list of slave's URLs
// tokens[5] -> name of the transformation function (and possibly parameters)
/*** create import object ***/
AbstractFactory importFactory = new ImportFactory();
Import importObj = null;
// tokenize input
string importPattern = @",|\s";
string[] importTokens = Regex.Split(tokens[1], importPattern).Where(s => s != String.Empty).ToArray <string>();
// list to collect possible file paths
List <string> filePathsList = new List <string>();
for (int i = 0; i < importTokens.Length; i++)
{
if (importTokens[i].StartsWith("OP")) // input comes from operator
{
importObj = importFactory.GetImport(new string[] { "OpImport" }, null, 0, 0);
break; // assuming only one operator
}
if (importTokens[i].Contains(".")) // input comes from file
{
filePathsList.Add(importTokens[i]);
}
else
{
Console.WriteLine("Neither operator nor input file!!!");
}
}
/*** create routing object ***/
AbstractFactory routingFactory = new RoutingFactory();
Route routeObj;
// tokenize routing policy
string[] routingTokens;
string routingPattern = @"[)(\s]";
if (input.RoutingType != null)
{
routingTokens = Regex.Split(input.RoutingType, routingPattern).Where(s => s != String.Empty).ToArray <string>();
}
else
{
routingTokens = Regex.Split("Output", routingPattern).Where(s => s != String.Empty).ToArray <string>();
}
routeObj = routingFactory.GetRouting(routingTokens, downstreamUrls, input.Semantic.ToLower());
/*** create processing object ***/
AbstractFactory processingFactory = new ProcessingFactory();
Process processObj;
// tokenize processing function
string processingPattern = @",|\s";
string[] processingTokens = Regex.Split(tokens[5], processingPattern).Where(s => s != String.Empty).ToArray <string>();
// if it's a count operator or a uniq - it needs state - for exactly-once
_stateful = Stateful(processingTokens[0]);
processObj = processingFactory.GetProcessing(processingTokens);
string[] plainUrls = urls.ToArray().Where(s => s != String.Empty).ToArray <string>();
bool wasNull = importObj == null;
foreach (string url in plainUrls)
{
if (wasNull)
{
// tokenize routing policy for the first operator
string[] merge = firstOP ? FileImportRouting(input.RoutingTypeToReadFromFile) : FileImportRouting(input.RoutingType);
// supports both import in the beginning or in the middle
if (input.RoutingType != null)
{
importObj = importFactory.GetImport(merge, filePathsList.ToArray(),
plainUrls.ToList().IndexOf(url), plainUrls.Length);
}
else
{
importObj = importFactory.GetImport(new string[] { "FileImport", "primary" }, filePathsList.ToArray(), plainUrls.ToList().IndexOf(url), plainUrls.Length);
}
}
System.Diagnostics.Process.Start(@"Slave.exe", SerializeObject(importObj) + " " + SerializeObject(routeObj) + " " +
SerializeObject(processObj) + " " + SerializeObject(url) + " " + SerializeObject(input.PuppetMasterUrl) + " " +
SerializeObject(input.IsLogFull) + " " + SerializeObject(input.Semantic.ToLower()) + " " +
SerializeObject(getSiblings(plainUrls, url)) + " " + SerializeObject(_stateful));
}
}
private void killProcess(ConnectionPack cp)
{
DADStormProcess.ClientProcess process = new DADStormProcess.ClientProcess(cp);
process.crash();
}
private void createNewOperator(String[] splitStr)
{
//foreach(string str in splitStr){
// System.Console.WriteLine (str);
//}
int counter = 0;
string current_operator_id = splitStr[0];
// ----------------- //
// INput Operators //
// ----------------- //
if (splitStr[1].Equals("input", StringComparison.OrdinalIgnoreCase) && splitStr[2].Equals("ops", StringComparison.OrdinalIgnoreCase))
{
counter = 3;
// then it must
}
else if (splitStr[1].Equals("input_ops", StringComparison.OrdinalIgnoreCase))
{
counter = 2;
}
while (!(splitStr[counter].Equals("rep", StringComparison.OrdinalIgnoreCase) && splitStr[counter + 1].Equals("fact", StringComparison.OrdinalIgnoreCase)) &&
!splitStr[counter].Equals("rep_fact", StringComparison.OrdinalIgnoreCase))
{
List <String> existingOperators;
String emitingOperator = splitStr[counter];
if (downStreamOperators.TryGetValue(emitingOperator, out existingOperators))
{
//found something
//Add current operator to downStreamOperators of inputOperator
existingOperators.Add(current_operator_id);
}
else
{
//was not initialized yet
existingOperators = new List <String>();
downStreamOperators.Add(emitingOperator, existingOperators);
//Add current operator to downStreamOperators of inputOperator
existingOperators.Add(current_operator_id);
}
counter++;
}
// ------------------ //
// Replication Factor //
// ------------------ //
if (splitStr[counter].Equals("rep", StringComparison.OrdinalIgnoreCase) && splitStr[counter + 1].Equals("fact", StringComparison.OrdinalIgnoreCase))
{
counter++;
counter++;
// then it must
}
else if (splitStr[counter].Equals("rep_fact", StringComparison.OrdinalIgnoreCase))
{
counter++;
}
string rep_factor = splitStr[counter++];
//ignoring word routing
counter++;
string routing = splitStr[counter++];
//ignoring word address
counter++;
List <ConnectionPack> currentConnectionPacks = new List <ConnectionPack>();
while (!(splitStr[counter].Equals("operator", StringComparison.OrdinalIgnoreCase) && splitStr[counter + 1].Equals("spec", StringComparison.OrdinalIgnoreCase)) &&
!splitStr[counter].Equals("operator_spec", StringComparison.OrdinalIgnoreCase))
{
string url = splitStr[counter];
string[] parsedUrl = url.Split(new[] { '/', ':' }, StringSplitOptions.RemoveEmptyEntries);
string ip = parsedUrl[1];
if (ip.Equals("localhost", StringComparison.OrdinalIgnoreCase))
{
ip = getMyIp();
}
ConnectionPack cp = new ConnectionPack(ip, Int32.Parse(parsedUrl[2]));
currentConnectionPacks.Add(cp);
counter++;
}
operatorsConPacks.Add(current_operator_id, currentConnectionPacks);
if (splitStr[counter].Equals("operator", StringComparison.OrdinalIgnoreCase) && splitStr[counter + 1].Equals("spec", StringComparison.OrdinalIgnoreCase))
{
counter++;
counter++;
// then it must
}
else if (splitStr[counter].Equals("operator_spec", StringComparison.OrdinalIgnoreCase))
{
counter++;
}
string operatorType = splitStr[counter++];
string dll = "CommonTypes.dll";
string className = "Default";
string methodName = null;
string[] staticAsrguments = null;
if (operatorType.Equals("CUSTOM", StringComparison.OrdinalIgnoreCase))
{
dll = splitStr[counter++];
className = splitStr[counter++];
methodName = splitStr[counter++];
}
else if (operatorType.Equals("UNIQ", StringComparison.OrdinalIgnoreCase))
{
methodName = "Uniq";
string[] args = { splitStr[counter++] };
staticAsrguments = args;
//field_number;
}
else if (operatorType.Equals("COUNT", StringComparison.OrdinalIgnoreCase))
{
methodName = "Count";
}
else if (operatorType.Equals("DUP", StringComparison.OrdinalIgnoreCase))
{
methodName = "Dup";
}
else if (operatorType.Equals("OUTPUT", StringComparison.OrdinalIgnoreCase))
{
methodName = "Output";
string[] args = { splitStr[counter++] }; //File to Output
staticAsrguments = args;
}
else if (operatorType.Equals("FILTER", StringComparison.OrdinalIgnoreCase))
{
methodName = "Filter";
string field_number = splitStr[counter++]; // field_number;
string condition = splitStr[counter++]; // condition;
string comparedValue = splitStr[counter++]; // value;
string[] args = { field_number, condition, comparedValue };
staticAsrguments = args;
}
//Create the Processes
foreach (ConnectionPack cp in currentConnectionPacks)
{
Daemon.ClientDaemon cd = new Daemon.ClientDaemon(new ConnectionPack(cp.Ip, daemonPort), fullLog);
cd.newThread(dll, className, methodName, cp.Port.ToString(), cp.Ip, semantics, routing, current_operator_id, staticAsrguments);
}
Thread.Sleep(100); //Make sure everything is created before we try anything else
logCommand("Operator:" + current_operator_id + " has " + currentConnectionPacks.Count + " replicas, created");
}
/// <summary>
/// Dos the first start connections.
/// </summary>
private void doFirstStartConnections()
{
if (firstStart)
{
System.Console.WriteLine();
PuppetDebug("Deploying Connections in network");
System.Console.WriteLine();
//Creating the network betwen all operators
foreach (KeyValuePair <string, List <string> > item in downStreamOperators)
{
List <ConnectionPack> outputingReplicas;
List <ConnectionPack> receivingReplicas;
string emitingOperator = item.Key;
//Getting list of Outputing replicas
if (operatorsConPacks.TryGetValue(emitingOperator, out outputingReplicas))
{
//Then it is an operator
//foreach output replica in outputOPerator
foreach (ConnectionPack outPack in outputingReplicas)
{
DADStormProcess.ClientProcess outReplica = new DADStormProcess.ClientProcess(outPack);
//foreach receivingOperator
foreach (string receiving_operator in item.Value)
{
//Getting list of receiving replicas of operator
if (operatorsConPacks.TryGetValue(receiving_operator, out receivingReplicas))
{
//for each replica in the receivingOperator
PuppetDebug("Connection: " + outPack + " outputs to: " + receiving_operator);
outReplica.addDownStreamOperator(receivingReplicas, receiving_operator);
}
}
}
foreach (string receiving_operator in item.Value)
{
//Getting list of receiving replicas of operator
if (operatorsConPacks.TryGetValue(receiving_operator, out receivingReplicas))
{
//for each replica in the receivingOperator
//Adding the reversed
foreach (ConnectionPack receivingRep in receivingReplicas)
{
PuppetDebug("Connection: receives " + receivingRep + " from: " + emitingOperator);
DADStormProcess.ClientProcess inReplica = new DADStormProcess.ClientProcess(receivingRep);
inReplica.addUpperStreamOperator(outputingReplicas, emitingOperator);
}
}
}
}
else
{
//else it must be a file
//foreach receivingOperator
foreach (string receiving_operator in item.Value)
{
addFileToOperator(item.Key, receiving_operator);
}
}
}
string ip = getMyIp();
ConnectionPack myConPack = new ConnectionPack(ip, port);
foreach (List <ConnectionPack> list in operatorsConPacks.Values)
{
foreach (ConnectionPack cp in list)
{
DADStormProcess.ClientProcess process = new DADStormProcess.ClientProcess(cp);
//Telling every operator's replica all its replicas
process.assignReplicaList(list);
//Telling every operator's replica where puppetMaster is
process.assignPuppetConPack(myConPack);
}
}
this.firstStart = false;
}
else
{
//we already started everything..
return;
}
}
private void ExecuteLine(string cmd)
{
PuppetMaster.GetInstance().Log("Running config file command: " + cmd);
if (cmd.StartsWith(" "))
{
MessageBox.Show("Invalid line to be processed!");
}
else if (cmd.StartsWith("Semantics"))
{
string[] pieces = cmd.Split(' ');
if (pieces.Length == 2)
{
_sematic = pieces[1];
}
}
else if (cmd.StartsWith("LoggingLevel"))
{
string[] pieces = cmd.Split(' ');
if (pieces.Length == 2 && pieces[1].Equals("full"))
{
_isLogFull = true;
}
}
else if (cmd.StartsWith("OP"))
{
// Operators setup
string[] res = cmd.Split(' ');
int opId = int.Parse(res[0].Substring(2));
List <string> listUrls = new List <string>();
for (int i = 10; i < res.Length && res[i].StartsWith("tcp://"); i++)
{
listUrls.Add(res[i].Replace(",", string.Empty));
}
string ip = GetOwnIp();
string myLogUrl = "tcp://" + ip + ":" + PuppetMaster.Port + "/" + PuppetMaster.RemoteName;
PuppetMaster.GetInstance().Log("IP Sent for PCS: " + myLogUrl);
if (_sematic.Equals(String.Empty))
{
_sematic = "at-most-once";
}
ConnectionPack thingsToSend = new ConnectionPack(cmd, _isLogFull, listUrls, myLogUrl, _sematic);
_whatShouldISentToOperators.Add(opId, thingsToSend);
if (res[3].StartsWith("OP") && !res[3].EndsWith(".dat"))
{
// Put in connection pack of the previous operator the urls of this one. So he can direct its output
int op2Id = int.Parse(res[3].Substring(2));
_whatShouldISentToOperators[op2Id].ReplicaUrlsOutput = listUrls;
_whatShouldISentToOperators[op2Id].RoutingType = res[8];
}
if (res[0].Equals("OP1"))
{
_whatShouldISentToOperators[opId].RoutingTypeToReadFromFile = res[8];
}
PuppetMaster.GetInstance().CreateOperator(opId, listUrls);
}
else
{
// Commands to operators
// Send every creation command to the operators
if (!_wereOperatorsCreated)
{
PuppetMaster.GetInstance().Log("Creating operators...");
foreach (KeyValuePair <int, ConnectionPack> entry in _whatShouldISentToOperators)
{
PcsManager.GetInstance().SendCommand(entry.Value);
}
_wereOperatorsCreated = true;
}
ParseCommand(cmd);
}
}
public void reborn(ConnectionPack deadGuy)
{
remoteProcess.reborn(deadGuy);
}
public override bool Equals(object obj)
{
ConnectionPack other = obj as ConnectionPack;
return(other.Port.Equals(this.Port) && other.Ip.Equals(this.Ip));
}
public string needsDivert(string tupleID, ConnectionPack nextOwner)
{
return(remoteProcess.needsDivert(tupleID, nextOwner));
}
public void IamAliveOnceAgain(ConnectionPack reborningGuy, string opID)
{
remoteProcess.IamAliveOnceAgain(reborningGuy, opID);
}
public void assignPuppetConPack(ConnectionPack puppetCp)
{
remoteProcess.assignPuppetConPack(puppetCp);
}
public bool isAlive(ConnectionPack brother)
{
return(ServerProcess.Instance.isAlive(brother));
}
public void IamAliveOnceAgain(ConnectionPack reborningGuy, string opID)
{
ServerProcess.Instance.IamAliveOnceAgain(reborningGuy, opID);
}
public string needsDivert(string tupleID, ConnectionPack nextOwner)
{
return(ServerProcess.Instance.needsDivert(tupleID, nextOwner));
}
