public static void RunRepairer()
{
string thishost = System.Net.Dns.GetHostName();
Random rnd = new Random(unchecked (DateTime.Now.Millisecond
+ System.Diagnostics.Process.GetCurrentProcess().Id
+ 5757981));
System.Threading.Thread.Sleep(7000);
if (!MySpace.DataMining.AELight.dfs.DfsConfigExists(
MySpace.DataMining.AELight.dfs.DFSXMLNAME))
{
_ThreadDisabled();
return;
}
MySpace.DataMining.AELight.dfs dc
= MySpace.DataMining.AELight.dfs.ReadDfsConfig_unlocked(
MySpace.DataMining.AELight.dfs.DFSXMLNAME);
if (null == dc.FileDaemon || !dc.FileDaemon.Enabled)
{
dc = null;
_ThreadDisabled();
return;
}
for (; ; System.Threading.Thread.Sleep(dc.FileDaemon.RepairSleep))
{
List <RepairItem> repairs = new List <RepairItem>(dc.FileDaemon.MaxRepairs);
{
List <RepairItem> allrepairs = new List <RepairItem>(200);
foreach (string host in dc.Slaves.SlaveList.Split(';'))
{
try
{
string netpath = MySpace.DataMining.AELight.Surrogate.NetworkPathForHost(host);
string fp = netpath + @"\filescan.log";
string[] repairfiles = System.IO.File.ReadAllLines(fp);
try
{
System.IO.File.Delete(fp);
}
catch
{
}
foreach (string rf in repairfiles)
{
int i = rf.IndexOf("] ");
if (-1 != i)
{
string s = rf.Substring(i + 2);
i = IndexOfEndOfParam(s);
string rfhost = s.Substring(0, i);
s = s.Substring(i + 1);
i = IndexOfEndOfParam(s);
string spos = s.Substring(0, i);
s = s.Substring(i + 1);
string chunkname = s.Trim('"');
RepairItem ri = new RepairItem();
ri.chunkname = chunkname;
ri.host = rfhost;
try
{
ri.filepos = int.Parse(spos); // spos might be "<done>"; if so, don't add it.
allrepairs.Add(ri);
}
catch
{
}
}
}
}
catch
{
}
}
allrepairs.Sort(new Comparison <RepairItem>(
delegate(RepairItem ri1, RepairItem ri2)
{
return(string.Compare(ri1.chunkname, ri2.chunkname, true));
}));
for (int i = 1; i < allrepairs.Count; i++)
{
if (0 == string.Compare(allrepairs[i - 1].chunkname, allrepairs[i].chunkname, true))
{
allrepairs[i].hits += allrepairs[i - 1].hits;
allrepairs[i - 1].hits = 0;
}
}
allrepairs.Sort(new Comparison <RepairItem>(
delegate(RepairItem ri1, RepairItem ri2)
{
return(ri2.hits - ri1.hits);
}));
for (int i = 0; i < 10 && i < allrepairs.Count && allrepairs[i].hits > 0; i++)
{
repairs.Add(allrepairs[i]);
}
}
foreach (RepairItem ri in repairs)
{
bool found = false;
foreach (MySpace.DataMining.AELight.dfs.DfsFile df in dc.Files)
{
foreach (MySpace.DataMining.AELight.dfs.DfsFile.FileNode fn in df.Nodes)
{
if (0 == string.Compare(fn.Name, ri.chunkname, true))
{
found = true;
string state = "no attempt";
string newhost = "N/A";
string[] ahosts = fn.Host.Split(';');
if (dc.FileDaemon.AutoRepair)
{
for (int itries = 0; itries < 5; itries++, System.Threading.Thread.Sleep(1000))
{
try
{
System.IO.File.AppendAllText("filerepairlog.txt",
"[" + DateTime.Now + "] NotReplacing \"" + ri.chunkpath
+ "\" with replicate chunk: AutoRepair is false"
+ Environment.NewLine);
break;
}
catch
{
}
}
break;
}
else if (ahosts.Length < 2 ||
ahosts.Length < dc.Replication)
{
//state = "no attempt: file replicate count is " + ahosts.Length
// + "; cluster replication factor is " + dc.Replication;
break;
}
else
{
foreach (string ahost in ahosts)
{
if (0 != string.Compare(ahost, ri.host, true))
{
try
{
string newchunkname = "zd.fdrepair." + Guid.NewGuid() + ".zd";
string newchunkpath = MySpace.DataMining.AELight.Surrogate.NetworkPathForHost(ri.host)
+ @"\" + newchunkname;
System.IO.File.Copy(
MySpace.DataMining.AELight.Surrogate.NetworkPathForHost(ahost) + @"\" + ri.chunkname,
newchunkpath
);
string chunkpath = ri.chunkpath;
System.IO.File.Delete(chunkpath);
System.IO.File.Move(newchunkpath, chunkpath);
state = "success";
newhost = ahost;
break;
}
catch (Exception e)
{
state = "failure: " + e.ToString();
}
}
}
}
for (int itries = 0; itries < 5; itries++, System.Threading.Thread.Sleep(1000))
{
try
{
System.IO.File.AppendAllText("filerepairlog.txt",
"[" + DateTime.Now + "] Replacing \"" + ri.chunkpath
+ "\" with chunk from " + newhost
+ ": " + state
+ Environment.NewLine);
break;
}
catch
{
}
}
break;
}
}
if (found)
{
break;
}
}
}
}
}
public static void RunNotifierService()
{
for (bool wait = true; ;)
{
if (wait)
{
System.Threading.Thread.Sleep(1000 * 30);
}
else
{
System.Threading.Thread.Sleep(1000 * 3);
}
wait = true;
{
if (!MySpace.DataMining.AELight.dfs.DfsConfigExists(
MySpace.DataMining.AELight.dfs.DFSXMLNAME))
{
// If not surrogate, wait a lot longer.
// Need to keep this loop in case this machine becomes surrogate.
System.Threading.Thread.Sleep(1000 * 30 * 9);
continue;
}
}
NotifyInfo ninfo = NotifyInfo.Load();
MySpace.DataMining.AELight.dfs dc = null; // Load once if needed.
{
DateTime starttime = DateTime.Now;
if (null == ninfo.LastHistoryTime)
{
ninfo.LastHistoryTime = starttime.ToString();
ninfo.Save();
}
else
{
bool AnyNotifyAll = false;
for (int i = 0; i < ninfo.Notify.Count; i++)
{
NotifyInfo.NEntry ne = ninfo.Notify[i];
if (-1 == ne.WaitOnJID)
{
AnyNotifyAll = true;
break;
}
}
if (null == ninfo.NewHistory)
{
if (AnyNotifyAll)
{
ninfo.NewHistory = new List <NotifyInfo.ConfigNewHistory>();
}
}
else
{
if (!AnyNotifyAll)
{
ninfo.NewHistory = null;
ninfo.LastHistoryTime = starttime.ToString();
ninfo.Save();
}
}
if (AnyNotifyAll)
{
DateTime lasthistory = DateTime.Parse(ninfo.LastHistoryTime);
try
{
using (System.IO.FileStream stmhist = new System.IO.FileStream("execlog.txt", System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite))
{
long stmhistLength = stmhist.Length;
bool skipfirst = false;
if (stmhistLength > 0x400 * 8)
{
stmhist.Position = stmhistLength - (0x400 * 8);
skipfirst = true;
}
System.IO.StreamReader srhist = new System.IO.StreamReader(stmhist);
if (skipfirst)
{
srhist.ReadLine(); // Probably partial line, ignore it.
}
string ln;
while (null != (ln = srhist.ReadLine()))
{
if (-1 != ln.IndexOf(" exec ", StringComparison.OrdinalIgnoreCase))
{
string user = null, stime = null, cmd = null, snhjid = null;
if (ParseExecLogLine(ln, ref user, ref stime, ref cmd, ref snhjid))
{
DateTime time = DateTime.Parse(stime);
if (time > lasthistory)
{
string secondarg = "";
{
int secondargstart = 0;
if (cmd.StartsWith("\""))
{
secondargstart = cmd.IndexOf('"', 1);
secondargstart++;
}
secondargstart = cmd.IndexOf(' ', secondargstart);
secondargstart++;
int secondargend = cmd.IndexOf(' ', secondargstart);
if (-1 != secondargend)
{
secondarg = cmd.Substring(secondargstart, secondargend - secondargstart).Trim('"');
}
}
if (0 == string.Compare("exec", secondarg, StringComparison.OrdinalIgnoreCase))
{
NotifyInfo.ConfigNewHistory cnh = new NotifyInfo.ConfigNewHistory();
cnh.JID = long.Parse(snhjid);
//cnh.User = user;
//cnh.Time = stime;
//cnh.Command = cmd;
cnh.History = ln;
ninfo.NewHistory.Add(cnh);
}
}
}
}
}
}
}
catch (System.IO.FileNotFoundException)
{
}
{
ninfo.LastHistoryTime = starttime.ToString();
ninfo.Save();
}
{
// Handle the 'notifyfinish <JobInfo>' cases now.
for (int inh = 0; inh < ninfo.NewHistory.Count; inh++)
{
NotifyInfo.ConfigNewHistory cnh = ninfo.NewHistory[inh];
if (!System.IO.File.Exists(cnh.JID.ToString() + ".jid"))
{
ninfo.NewHistory.RemoveAt(inh);
inh--;
ninfo.Save();
for (int i = 0; i < ninfo.Notify.Count; i++)
{
NotifyInfo.NEntry ne = ninfo.Notify[i];
if (-1 == ne.WaitOnJID)
{
if (System.Text.RegularExpressions.Regex.IsMatch(cnh.History, ne.WaitOnHistoryRegex,
System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
try
{
string clustername = "N/A";
string SMTP;
{
if (dc == null)
{
dc = MySpace.DataMining.AELight.dfs.ReadDfsConfig_unlocked(
MySpace.DataMining.AELight.dfs.DFSXMLNAME);
}
if (dc.ClusterName != null)
{
clustername = dc.ClusterName;
}
SMTP = dc.SMTP;
}
if (null == SMTP)
{
throw new Exception("SMTP server is null");
}
string subject = "Qizmt Notification";
string body;
{
StringBuilder sbbody = new StringBuilder();
sbbody.AppendLine(ne.GetFinishedUserMessage());
sbbody.AppendFormat("{1} - Cluster Name{0}", Environment.NewLine, clustername);
//sbbody.AppendFormat("{1} - Command{0}", Environment.NewLine, cnh.Command); // Insecure.
sbbody.AppendFormat("{1} - Job Identifier{0}", Environment.NewLine, cnh.JID);
sbbody.AppendFormat("{1} - Notify Identifier{0}", Environment.NewLine, ne.ID);
sbbody.AppendFormat("{1} - Completion Time{0}", Environment.NewLine, DateTime.Now);
sbbody.AppendFormat("{1} - User Added Notify{0}", Environment.NewLine, ne.GetPlainUserAdded());
body = sbbody.ToString();
}
SendQNEmail(SMTP, ne.Email, subject, body);
}
catch (Exception e)
{
throw new Exception("Problem sending email to " + ne.Email + " after JID " + ne.WaitOnJID + " finished (notifyfinish <JobInfo>)", e);
}
System.Threading.Thread.Sleep(1000 * 3);
}
}
}
wait = false;
break;
}
}
}
}
}
for (int i = 0; i < ninfo.Notify.Count; i++)
{
NotifyInfo.NEntry ne = ninfo.Notify[i];
if (-1 == ne.WaitOnJID)
{
}
else if (!System.IO.File.Exists(ne.WaitOnJID.ToString() + ".jid"))
{
ninfo.Notify.RemoveAt(i);
i--;
ninfo.Save();
try
{
string clustername = "N/A";
string SMTP;
{
if (dc == null)
{
dc = MySpace.DataMining.AELight.dfs.ReadDfsConfig_unlocked(
MySpace.DataMining.AELight.dfs.DFSXMLNAME);
}
if (dc.ClusterName != null)
{
clustername = dc.ClusterName;
}
SMTP = dc.SMTP;
}
if (null == SMTP)
{
throw new Exception("SMTP server is null");
}
string subject = "Qizmt Notification";
string body;
{
StringBuilder sbbody = new StringBuilder();
sbbody.AppendLine(ne.GetFinishedUserMessage());
sbbody.AppendFormat("{1} - Cluster Name{0}", Environment.NewLine, clustername);
sbbody.AppendFormat("{1} - Job Identifier{0}", Environment.NewLine, ne.WaitOnJID);
sbbody.AppendFormat("{1} - Notify Identifier{0}", Environment.NewLine, ne.ID);
sbbody.AppendFormat("{1} - Completion Time{0}", Environment.NewLine, DateTime.Now);
sbbody.AppendFormat("{1} - User Added Notify{0}", Environment.NewLine, ne.GetPlainUserAdded());
body = sbbody.ToString();
}
SendQNEmail(SMTP, ne.Email, subject, body);
}
catch (Exception e)
{
throw new Exception("Problem sending email to " + ne.Email + " after JID " + ne.WaitOnJID + " finished (notifyfinish <JobID>)", e);
}
wait = false;
break;
}
}
}
}
}
