private void RunCloudMake(Vsync.Group vsyncGroup)
{
_dependencyStructure.Run();
if (_rank == 0)
{
Dist(vsyncGroup, _dependencyStructure.GetModifiedConfigFiles(),
_dependencyStructure.GetModifiedCloudMakeConfigFiles());
}
else
{
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank - 1]), CloudManager.RUN_CLOUD_MAKE);
}
_dependencyStructure.Reset();
}
private void ModifiedState(Vsync.Group vsyncGroup, string path, string name)
{
XmlDocument xmlDoc = new XmlDocument();
string filename = _name + "/" + name;
string content;
xmlDoc.Load(path);
content = xmlDoc.OuterXml;
if (_localCloudMakefile.Compatible(filename).Count > 0)
{
UpdateState(vsyncGroup, filename, content);
}
for (int i = 0; i < _leaderCloudMakefiles.Count; i++)
{
if (_leaderCloudMakefiles [i].Compatible(filename).Count > 0)
{
vsyncGroup.P2PSend(new Address(_leaders [i] [0]), CloudManager.UPDATE_STATE, filename, content);
}
}
}
private void ProcessMessages(Vsync.Group vsyncGroup, List <Tuple <MessageType, List <string> > > msgs)
{
foreach (Tuple <MessageType, List <string> > msg in msgs)
{
MessageType type = msg.Item1;
MessageType tempType;
List <string> parameters = msg.Item2;
List <string> tempParameters = null;
string vsyncAddress, memberName, filename, xmlString, procName, configFilename;
int minPartition, partition, last;
WriteMessage(type, parameters);
switch (type)
{
case MessageType.NEW_NODE:
vsyncAddress = parameters [0];
CloudManager.WriteLine(_debugFilename, vsyncAddress.ToString());
vsyncGroup.P2PSend(new Address(vsyncAddress), CloudManager.LEADER_INFO, _vsyncAddress,
_partition, _rank);
break;
case MessageType.FAIL_NODE:
vsyncAddress = parameters [0];
partition = CloudManager.FindPartition(_leaders, _reserveLeaders, vsyncAddress);
if (partition >= 0)
{
throw new NotImplementedException();
}
else if (partition == -1)
{
_reserveLeaders.Remove(vsyncAddress);
}
else if (_addressToName.ContainsKey(vsyncAddress))
{
memberName = _addressToName [vsyncAddress];
_addressToName.Remove(vsyncAddress);
_nameToAddress.Remove(memberName);
if ((_state == State.ACTIVE) && IsEventInPartition(memberName + "/fail_node"))
{
if (_rank == 0)
{
if (_leaders [_partition].Count == 1)
{
MemberLeave(memberName);
_updatesDelivered += 1;
}
else
{
_pending.Add(msg);
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank + 1]),
CloudManager.MEMBER_LEAVE, memberName);
}
}
}
}
break;
case MessageType.MEMBER_JOIN:
vsyncAddress = parameters [0];
memberName = parameters [1];
_addressToName.Add(vsyncAddress, memberName);
_nameToAddress.Add(memberName, vsyncAddress);
if ((_partition >= 0) && (_rank == 0) && (_rank == _leaders [_partition].Count - 1))
{
NewMember(memberName);
_updatesDelivered += 1;
}
else if ((_partition >= 0) && (_rank == 0))
{
tempType = MessageType.NEW_MEMBER;
tempParameters = new List <string> ();
tempParameters.Add(memberName);
_pending.Add(new Tuple <MessageType, List <string> > (tempType, tempParameters));
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank + 1]), CloudManager.NEW_MEMBER,
memberName);
}
break;
case MessageType.MEMBER_LEAVE:
if (!(_state == State.ACTIVE))
{
throw new Exception("CloudMakeLeader: Received MEMBER_LEAVE without being active.");
}
memberName = parameters [0];
if (_rank == _leaders [_partition].Count - 1)
{
MemberLeave(memberName);
_updatesDelivered += 1;
if (_rank > 0)
{
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank - 1]), CloudManager.ACK);
}
}
else
{
_pending.Add(msg);
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank + 1]), CloudManager.MEMBER_LEAVE,
memberName);
}
break;
case MessageType.LEADER_JOIN:
vsyncAddress = parameters [0];
minPartition = CloudManager.FindMinPartition(_leaders);
if ((minPartition >= 0) && (_leaders [minPartition].Count < _nreplicas))
{
last = _leaders [minPartition].Count - 1;
_leaders [minPartition].Add(vsyncAddress);
CloudManager.WriteLine(_debugFilename, "Previous Leader: " + _leaders [minPartition] [last]);
CloudManager.WriteLine(_debugFilename, "Me: " + _vsyncAddress);
if ((_state == State.ACTIVE) && (_leaders [minPartition] [last] == _vsyncAddress))
{
Tuple <List <string>, List <string>, List <string> > triple = GetState();
vsyncGroup.P2PSend(new Address(_leaders [minPartition] [last + 1]),
CloudManager.STATE_TRANSFER, triple.Item1, triple.Item2, triple.Item3);
}
}
else
{
_reserveLeaders.Add(vsyncAddress);
}
break;
case MessageType.NEW_MEMBER:
memberName = parameters [0];
CloudManager.WriteLine(_debugFilename, "Partition: " + _partition.ToString());
if (_rank == _leaders [_partition].Count - 1)
{
CloudManager.WriteLine(_debugFilename, "Call NewMember.");
NewMember(memberName);
_updatesDelivered += 1;
if (_rank > 0)
{
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank - 1]), CloudManager.ACK);
}
}
else
{
_pending.Add(msg);
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank + 1]), CloudManager.NEW_MEMBER,
memberName);
}
break;
case MessageType.NEW_CONFIG:
configFilename = parameters [0];
CloudManager.WriteLine(_debugFilename, "Partition: " + _partition.ToString());
if (_rank == _leaders [_partition].Count - 1)
{
CloudManager.WriteLine(_debugFilename, "Call NewConfig.");
NewConfig(configFilename);
CloudManager.WriteLine(_debugFilename, "Finished NewConfig.");
_updatesDelivered += 1;
if (_rank > 0)
{
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank - 1]), CloudManager.ACK);
}
}
else
{
_pending.Add(msg);
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank + 1]), CloudManager.NEW_CONFIGS,
parameters);
}
break;
case MessageType.STOPPED_PROC:
if (_state != State.ACTIVE)
{
throw new Exception("CloudMakeLeader: Received STOPPED_PROC without being active.");
}
memberName = parameters [0];
procName = parameters [1];
if (_rank == _leaders [_partition].Count - 1)
{
StoppedProc(memberName, procName);
_updatesDelivered += 1;
if (_rank > 0)
{
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank - 1]), CloudManager.ACK);
}
}
else
{
_pending.Add(msg);
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank + 1]), CloudManager.STOPPED_PROC,
memberName, procName);
}
break;
case MessageType.UPDATE_STATE:
if (_state != State.ACTIVE)
{
throw new Exception("CloudMakeLeader: Received UPDATE_STATE without being active");
}
filename = parameters [0];
xmlString = parameters [1];
if (_rank == _leaders [_partition].Count - 1)
{
UpdateState(filename, xmlString);
_updatesDelivered += 1;
if (_rank > 0)
{
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank - 1]), CloudManager.ACK);
}
}
else
{
_pending.Add(msg);
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank + 1]), CloudManager.UPDATE_STATE,
filename, xmlString);
}
break;
case MessageType.ACK:
tempType = _pending [0].Item1;
tempParameters = _pending [0].Item2;
switch (tempType)
{
case MessageType.NEW_MEMBER:
CloudManager.WriteLine(_debugFilename, "Call NewMember.");
memberName = tempParameters [0];
NewMember(memberName);
break;
case MessageType.NEW_CONFIG:
configFilename = tempParameters [0];
CloudManager.WriteLine(_debugFilename, "Call NewConfig.");
NewConfig(configFilename);
CloudManager.WriteLine(_debugFilename, "Finished NewConfig.");
break;
case MessageType.MEMBER_LEAVE:
memberName = tempParameters [0];
MemberLeave(memberName);
break;
case MessageType.STOPPED_PROC:
memberName = tempParameters [0];
procName = tempParameters [1];
StoppedProc(memberName, procName);
break;
case MessageType.UPDATE_STATE:
filename = tempParameters [0];
xmlString = tempParameters [1];
UpdateState(filename, xmlString);
break;
default:
throw new Exception("CloudMakeLeader: Unknown Message Type " + _pending [0].Item1.ToString() +
" in pending queue.");
}
_updatesDelivered += 1;
_pending.RemoveAt(0);
if (_rank > 0)
{
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank - 1]), CloudManager.ACK);
}
break;
case MessageType.RUN_CLOUD_MAKE:
RunCloudMake(vsyncGroup);
break;
case MessageType.STATE_TRANSFER:
if (_state != State.WAITFORSTATE)
{
throw new Exception("CloudMakeLeader: Received STATE_TRANSFER without being at WAITFORSTATE " +
"state");
}
List <string> directories = new List <string> ();
List <string> xmlFilenames = new List <string> ();
List <string> xmlContents = new List <string> ();
int i = 0;
while ((parameters.Count > 2 * i + 1) && (parameters [2 * i + 1] == ""))
{
string dir = parameters [2 * i];
Directory.CreateDirectory(dir);
directories.Add(dir);
#if DEBUG
CloudManager.WriteLine(_debugFilename, "State Transfer Directory: " + dir);
#endif
i++;
}
while (parameters.Count > 2 * i + 1)
{
string xmlFilename = parameters [2 * i];
string xmlContent = parameters [2 * i + 1];
XmlDocument xmlDoc = new XmlDocument();
xmlFilenames.Add(xmlFilename);
xmlContents.Add(xmlContent);
#if DEBUG
CloudManager.WriteLine(_debugFilename, "State Transfer Filename: " + xmlFilename);
#endif
xmlDoc.LoadXml(xmlContent);
xmlDoc.Save(xmlFilename);
i++;
}
_state = State.ACTIVE;
if (_rank != _leaders [_partition].Count - 1)
{
vsyncGroup.P2PSend(new Address(_leaders [_partition] [_rank + 1]),
CloudManager.STATE_TRANSFER, directories, xmlFilenames, xmlContents);
}
break;
case MessageType.SEND_PENDING:
throw new NotImplementedException();
default:
throw new Exception("CloudMakeLeader: I should not have received " + msg.ToString() +
" message.");
}
CloudManager.WriteLine(_debugFilename, "Pending:");
foreach (Tuple <MessageType, List <string> > tempMsg in _pending)
{
WriteMessage(tempMsg.Item1, tempMsg.Item2);
}
}
}
private void Dist(Vsync.Group vsyncGroup, HashSet <string> activeConfigFiles,
HashSet <string> activeCloudMakeConfigFiles)
{
Tuple <HashSet <string>, List <RemoteProcessInfo> > processedInfo =
ProcessCloudMakeConfigFiles(vsyncGroup, activeCloudMakeConfigFiles);
HashSet <string> additionalActiveConfigFiles = new HashSet <string> ();
List <RemoteProcessInfo> remoteProcesses = processedInfo.Item2;
foreach (string activeConfigFile in processedInfo.Item1)
{
if (File.Exists(activeConfigFile))
{
additionalActiveConfigFiles.Add(activeConfigFile);
}
}
#if DEBUG
CloudManager.WriteLine(_debugFilename, "Active config files:");
foreach (string filename in activeConfigFiles)
{
CloudManager.WriteLine(_debugFilename, filename);
}
CloudManager.WriteLine(_debugFilename, "");
CloudManager.WriteLine(_debugFilename, "Active CloudMake config files:");
foreach (string filename in activeCloudMakeConfigFiles)
{
CloudManager.WriteLine(_debugFilename, filename);
}
CloudManager.WriteLine(_debugFilename, "");
#endif
activeConfigFiles.UnionWith(additionalActiveConfigFiles);
#if DEBUG
CloudManager.WriteLine(_debugFilename, "Files to Distribute:");
foreach (string filename in activeConfigFiles)
{
CloudManager.WriteLine(_debugFilename, filename);
}
CloudManager.WriteLine(_debugFilename, "");
#endif
foreach (string activeConfigFile in activeConfigFiles)
{
List <string> fields = new List <string> (activeConfigFile.Split('/'));
string nodename = fields [0];
List <string> fieldsList = fields.GetRange(1, fields.Count - 1);
string path = String.Join("/", fieldsList);
XmlDocument xmlDoc = new XmlDocument();
string filename = String.Join(Path.DirectorySeparatorChar.ToString(), fields);
string content;
List <string> pathList = new List <string> ();
List <string> contentList = new List <string> ();
xmlDoc.Load(filename);
content = xmlDoc.OuterXml;
#if DEBUG
CloudManager.WriteLine(_debugFilename, "Distribute the content of " + filename + " to " + nodename);
#endif
pathList.Add(path);
contentList.Add(content);
if (_nameToAddress.ContainsKey(nodename))
{
vsyncGroup.P2PSend(new Address(_nameToAddress [nodename]), CloudManager.UPDATE_CONFIGS, pathList,
contentList);
}
}
foreach (RemoteProcessInfo proc in remoteProcesses)
{
List <string> nameList = new List <string> ();
List <string> pathList = new List <string> ();
List <string> execList = new List <string> ();
List <string> argsList = new List <string> ();
string nodename;
if (proc.GetAction() == Action.RUN)
{
nameList.Add(proc.GetName());
pathList.Add(proc.GetPath());
execList.Add(proc.GetExec());
argsList.Add(proc.GetArgs());
nodename = proc.GetNodename();
if (_nameToAddress.ContainsKey(nodename))
{
vsyncGroup.P2PSend(new Address(_nameToAddress [nodename]), CloudManager.RUN_PROCESSES,
nameList, pathList, execList, argsList);
}
}
else
{
nameList.Add(proc.GetName());
nodename = proc.GetNodename();
if (_nameToAddress.ContainsKey(nodename))
{
vsyncGroup.P2PSend(new Address(_nameToAddress [nodename]), CloudManager.KILL_PROCESSES,
nameList);
}
}
}
}
ProcessCloudMakeConfigFiles(Vsync.Group vsyncGroup, HashSet <string> cloudMakeConfigFiles)
{
List <RemoteProcessInfo> remoteProcesses = new List <RemoteProcessInfo> ();
List <string> statePathList = new List <string> ();
List <string> stateNameList = new List <string> ();
Dictionary <int, HashSet <string> > configFiles = new Dictionary <int, HashSet <string> > ();
foreach (string cloudMakeConfigFile in cloudMakeConfigFiles)
{
XmlDocument xmlDoc = new XmlDocument();
XmlNode xmlNode;
string nodename = cloudMakeConfigFile.Substring(0, cloudMakeConfigFile.IndexOf('/'));
xmlDoc.Load(cloudMakeConfigFile);
xmlNode = xmlDoc.DocumentElement;
foreach (XmlNode childNode in xmlNode.ChildNodes)
{
switch (childNode.Name)
{
case ("process"):
if (childNode.Attributes ["action"].Value == "Run")
{
string name = childNode ["name"].InnerText;
string path = childNode ["path"].InnerText;
string exec = childNode ["exec"].InnerText;
string args = childNode ["args"].InnerText;
#if DEBUG
CloudManager.WriteLine(_debugFilename, "Run process " + name + ".");
#endif
remoteProcesses.Add(new RemoteProcessInfo(Action.RUN, nodename, name, path, exec, args));
}
else if (childNode.Attributes ["action"].Value == "Kill")
{
string name = childNode ["name"].InnerText;
#if DEBUG
CloudManager.WriteLine(_debugFilename, "Kill process " + name + ".");
#endif
remoteProcesses.Add(new RemoteProcessInfo(Action.KILL, nodename, name));
}
break;
case ("stateFile"):
string statePath = childNode ["path"].InnerText;
string stateName = childNode ["name"].InnerText;
statePathList.Add(statePath);
stateNameList.Add(stateName);
break;
case ("configFile"):
string configPath = childNode ["path"].InnerText;
string configName = childNode ["name"].InnerText;
string configFilename = (configPath == "") ? nodename + "/" + configName : nodename + "/" +
configPath + "/" + configName;
for (int i = 0; i < _leaderCloudMakefiles.Count; i++)
{
if (_leaderCloudMakefiles [i].Compatible(configFilename).Count > 0)
{
if (!configFiles.ContainsKey(i))
{
configFiles [i] = new HashSet <string> ();
}
configFiles [i].Add(configFilename);
}
}
if ((_localCloudMakefiles.ContainsKey(nodename)) &&
(_localCloudMakefiles [nodename].Compatible(configFilename).Count > 0))
{
List <string> parameters = new List <string> ();
parameters.Add(configFilename);
vsyncGroup.P2PSend(new Address(_nameToAddress [nodename]), CloudManager.NEW_CONFIGS,
parameters);
}
break;
default:
#if DEBUG
CloudManager.WriteLine(_debugFilename,
"CloudMakeLeader: Operation " + childNode.Name + " is not supported from CloudMakefile.");
#endif
break;
}
}
foreach (int i in configFiles.Keys)
{
if (i == _partition)
{
foreach (string configFilename in configFiles[i])
{
NewConfig(configFilename);
}
}
else
{
vsyncGroup.P2PSend(new Address(_leaders [i] [0]), CloudManager.NEW_CONFIGS,
new List <string> (configFiles [i]));
}
}
if (statePathList.Count > 0)
{
vsyncGroup.P2PSend(new Address(_nameToAddress [nodename]), CloudManager.ASK_STATES, statePathList,
stateNameList);
}
}
if (configFiles.ContainsKey(_partition))
{
return(new Tuple <HashSet <string>, List <RemoteProcessInfo> > (configFiles [_partition], remoteProcesses));
}
return(new Tuple <HashSet <string>, List <RemoteProcessInfo> > (new HashSet <string> (), remoteProcesses));
}
private void ProcessMessages(Vsync.Group vsyncGroup, List <Tuple <MessageType, List <string> > > msgs)
{
foreach (Tuple <MessageType, List <string> > msg in msgs)
{
MessageType type = msg.Item1;
List <string> parameters = msg.Item2;
string vsyncAddress, procName, path, exec, args, filename, content, configFilename;
#if DEBUG
WriteMessage(type, parameters);
#endif
switch (type)
{
case MessageType.NEW_NODE:
if (parameters.Count != 1)
{
throw new Exception("CloudMakeLocal: NEW_NODE: Does not accept " +
parameters.Count.ToString() + " parameters.");
}
vsyncAddress = parameters [0];
vsyncGroup.P2PSend(new Address(vsyncAddress), CloudManager.MEMBER_INFO, _vsyncAddress, _name);
break;
case MessageType.FAIL_NODE:
break;
case MessageType.MEMBER_JOIN:
MemberJoin(vsyncGroup);
break;
case MessageType.LEADER_JOIN:
if (parameters.Count != 1)
{
throw new Exception("CloudMakeLocal: LEADER_JOIN: Does not accept " +
parameters.Count.ToString() + " parameters.");
}
vsyncAddress = parameters [0];
CloudManager.AddLeader(_leaders, _reserveLeaders, _nreplicas, vsyncAddress);
break;
case MessageType.NEW_CONFIG:
if (parameters.Count != 1)
{
throw new Exception("CloudMakeLocal: NEW CONFIG: Should only have one configFilename.");
}
configFilename = parameters [0];
NewConfig(configFilename);
break;
case MessageType.RUN_PROCESS:
procName = parameters [0];
path = parameters [1];
exec = parameters [2];
args = parameters [3];
RunProcess(procName, path, exec, args);
break;
case MessageType.KILL_PROCESS:
procName = parameters [0];
KillProcess(procName);
break;
case MessageType.UPDATE_CONFIG:
filename = parameters [0];
content = parameters [1];
UpdateConfig(filename, content);
break;
case MessageType.ASK_STATE:
path = parameters [0];
filename = parameters [1];
AskState(vsyncGroup, path, filename);
break;
default:
throw new Exception("CloudMakeLocal: I should not have received " + msg.ToString() +
" message.");
}
}
}
private Tuple <HashSet <string>, List <RemoteProcessInfo> > ProcessCloudMakeConfigFile(Vsync.Group vsyncGroup)
{
string cloudMakeConfigFilename = _name + "/CloudMake/config.xml";
XmlDocument xmlDoc = new XmlDocument();
XmlNode xmlNode;
List <RemoteProcessInfo> processes = new List <RemoteProcessInfo> ();
List <string> statePathList = new List <string> ();
List <string> stateFilenameList = new List <string> ();
Dictionary <int, HashSet <string> > configFiles = new Dictionary <int, HashSet <string> > ();
HashSet <string> localConfigFiles = new HashSet <string> ();
xmlDoc.Load(cloudMakeConfigFilename);
xmlNode = xmlDoc.DocumentElement;
foreach (XmlNode childNode in xmlNode.ChildNodes)
{
switch (childNode.Name)
{
case ("process"):
if (childNode.Attributes ["action"].Value == "Run")
{
string name = childNode ["name"].InnerText;
string path = childNode ["path"].InnerText;
string exec = childNode ["exec"].InnerText;
string args = childNode ["args"].InnerText;
#if DEBUG
CloudManager.WriteLine(_debugFilename, "Run process " + name + ".");
#endif
processes.Add(new RemoteProcessInfo(Action.RUN, _name, name, path, exec, args));
}
else if (childNode.Attributes ["action"].Value == "Kill")
{
string name = childNode ["name"].InnerText;
#if DEBUG
CloudManager.WriteLine(_debugFilename, "Kill process " + name + ".");
#endif
processes.Add(new RemoteProcessInfo(Action.KILL, _name, name));
}
break;
case ("stateFile"):
string statePath = childNode ["path"].InnerText;
string stateName = childNode ["name"].InnerText;
if (statePath == "")
{
statePath = _curDirectory;
}
else
{
statePath = _curDirectory + Path.DirectorySeparatorChar.ToString() + statePath;
}
statePathList.Add(statePath);
stateFilenameList.Add(stateName);
break;
case ("configFile"):
string configPath = childNode ["path"].InnerText;
string configName = childNode ["name"].InnerText;
string configFilename = (configPath == "") ? _name + "/" + configName : _name + "/" +
configPath + "/" + configName;
for (int i = 0; i < _leaderCloudMakefiles.Count; i++)
{
if (_leaderCloudMakefiles [i].Compatible(configFilename).Count > 0)
{
if (!configFiles.ContainsKey(i))
{
configFiles [i] = new HashSet <string> ();
}
configFiles [i].Add(configFilename);
}
}
if (_localCloudMakefile.Compatible(configFilename).Count > 0)
{
NewConfig(configFilename);
localConfigFiles.Add(configFilename);
}
break;
default:
throw new Exception("CloudMakeLocal Config: Operation " + childNode.Name + " is not supported " +
"from CloudMake.");
}
}
foreach (int i in configFiles.Keys)
{
vsyncGroup.P2PSend(new Address(_leaders [i] [0]), CloudManager.NEW_CONFIGS,
new List <string> (configFiles [i]));
}
for (int i = 0; i < statePathList.Count; i++)
{
AskState(vsyncGroup, statePathList [i], stateFilenameList [i]);
}
return(new Tuple <HashSet <string>, List <RemoteProcessInfo> > (localConfigFiles, processes));
}
