public static void RunRepairer() { string thishost = System.Net.Dns.GetHostName(); Random rnd = new Random(unchecked (DateTime.Now.Millisecond + System.Diagnostics.Process.GetCurrentProcess().Id + 5757981)); System.Threading.Thread.Sleep(7000); if (!MySpace.DataMining.AELight.dfs.DfsConfigExists( MySpace.DataMining.AELight.dfs.DFSXMLNAME)) { _ThreadDisabled(); return; } MySpace.DataMining.AELight.dfs dc = MySpace.DataMining.AELight.dfs.ReadDfsConfig_unlocked( MySpace.DataMining.AELight.dfs.DFSXMLNAME); if (null == dc.FileDaemon || !dc.FileDaemon.Enabled) { dc = null; _ThreadDisabled(); return; } for (; ; System.Threading.Thread.Sleep(dc.FileDaemon.RepairSleep)) { List <RepairItem> repairs = new List <RepairItem>(dc.FileDaemon.MaxRepairs); { List <RepairItem> allrepairs = new List <RepairItem>(200); foreach (string host in dc.Slaves.SlaveList.Split(';')) { try { string netpath = MySpace.DataMining.AELight.Surrogate.NetworkPathForHost(host); string fp = netpath + @"\filescan.log"; string[] repairfiles = System.IO.File.ReadAllLines(fp); try { System.IO.File.Delete(fp); } catch { } foreach (string rf in repairfiles) { int i = rf.IndexOf("] "); if (-1 != i) { string s = rf.Substring(i + 2); i = IndexOfEndOfParam(s); string rfhost = s.Substring(0, i); s = s.Substring(i + 1); i = IndexOfEndOfParam(s); string spos = s.Substring(0, i); s = s.Substring(i + 1); string chunkname = s.Trim('"'); RepairItem ri = new RepairItem(); ri.chunkname = chunkname; ri.host = rfhost; try { ri.filepos = int.Parse(spos); // spos might be "<done>"; if so, don't add it. allrepairs.Add(ri); } catch { } } } } catch { } } allrepairs.Sort(new Comparison <RepairItem>( delegate(RepairItem ri1, RepairItem ri2) { return(string.Compare(ri1.chunkname, ri2.chunkname, true)); })); for (int i = 1; i < allrepairs.Count; i++) { if (0 == string.Compare(allrepairs[i - 1].chunkname, allrepairs[i].chunkname, true)) { allrepairs[i].hits += allrepairs[i - 1].hits; allrepairs[i - 1].hits = 0; } } allrepairs.Sort(new Comparison <RepairItem>( delegate(RepairItem ri1, RepairItem ri2) { return(ri2.hits - ri1.hits); })); for (int i = 0; i < 10 && i < allrepairs.Count && allrepairs[i].hits > 0; i++) { repairs.Add(allrepairs[i]); } } foreach (RepairItem ri in repairs) { bool found = false; foreach (MySpace.DataMining.AELight.dfs.DfsFile df in dc.Files) { foreach (MySpace.DataMining.AELight.dfs.DfsFile.FileNode fn in df.Nodes) { if (0 == string.Compare(fn.Name, ri.chunkname, true)) { found = true; string state = "no attempt"; string newhost = "N/A"; string[] ahosts = fn.Host.Split(';'); if (dc.FileDaemon.AutoRepair) { for (int itries = 0; itries < 5; itries++, System.Threading.Thread.Sleep(1000)) { try { System.IO.File.AppendAllText("filerepairlog.txt", "[" + DateTime.Now + "] NotReplacing \"" + ri.chunkpath + "\" with replicate chunk: AutoRepair is false" + Environment.NewLine); break; } catch { } } break; } else if (ahosts.Length < 2 || ahosts.Length < dc.Replication) { //state = "no attempt: file replicate count is " + ahosts.Length // + "; cluster replication factor is " + dc.Replication; break; } else { foreach (string ahost in ahosts) { if (0 != string.Compare(ahost, ri.host, true)) { try { string newchunkname = "zd.fdrepair." + Guid.NewGuid() + ".zd"; string newchunkpath = MySpace.DataMining.AELight.Surrogate.NetworkPathForHost(ri.host) + @"\" + newchunkname; System.IO.File.Copy( MySpace.DataMining.AELight.Surrogate.NetworkPathForHost(ahost) + @"\" + ri.chunkname, newchunkpath ); string chunkpath = ri.chunkpath; System.IO.File.Delete(chunkpath); System.IO.File.Move(newchunkpath, chunkpath); state = "success"; newhost = ahost; break; } catch (Exception e) { state = "failure: " + e.ToString(); } } } } for (int itries = 0; itries < 5; itries++, System.Threading.Thread.Sleep(1000)) { try { System.IO.File.AppendAllText("filerepairlog.txt", "[" + DateTime.Now + "] Replacing \"" + ri.chunkpath + "\" with chunk from " + newhost + ": " + state + Environment.NewLine); break; } catch { } } break; } } if (found) { break; } } } } }
public static void RunNotifierService() { for (bool wait = true; ;) { if (wait) { System.Threading.Thread.Sleep(1000 * 30); } else { System.Threading.Thread.Sleep(1000 * 3); } wait = true; { if (!MySpace.DataMining.AELight.dfs.DfsConfigExists( MySpace.DataMining.AELight.dfs.DFSXMLNAME)) { // If not surrogate, wait a lot longer. // Need to keep this loop in case this machine becomes surrogate. System.Threading.Thread.Sleep(1000 * 30 * 9); continue; } } NotifyInfo ninfo = NotifyInfo.Load(); MySpace.DataMining.AELight.dfs dc = null; // Load once if needed. { DateTime starttime = DateTime.Now; if (null == ninfo.LastHistoryTime) { ninfo.LastHistoryTime = starttime.ToString(); ninfo.Save(); } else { bool AnyNotifyAll = false; for (int i = 0; i < ninfo.Notify.Count; i++) { NotifyInfo.NEntry ne = ninfo.Notify[i]; if (-1 == ne.WaitOnJID) { AnyNotifyAll = true; break; } } if (null == ninfo.NewHistory) { if (AnyNotifyAll) { ninfo.NewHistory = new List <NotifyInfo.ConfigNewHistory>(); } } else { if (!AnyNotifyAll) { ninfo.NewHistory = null; ninfo.LastHistoryTime = starttime.ToString(); ninfo.Save(); } } if (AnyNotifyAll) { DateTime lasthistory = DateTime.Parse(ninfo.LastHistoryTime); try { using (System.IO.FileStream stmhist = new System.IO.FileStream("execlog.txt", System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite)) { long stmhistLength = stmhist.Length; bool skipfirst = false; if (stmhistLength > 0x400 * 8) { stmhist.Position = stmhistLength - (0x400 * 8); skipfirst = true; } System.IO.StreamReader srhist = new System.IO.StreamReader(stmhist); if (skipfirst) { srhist.ReadLine(); // Probably partial line, ignore it. } string ln; while (null != (ln = srhist.ReadLine())) { if (-1 != ln.IndexOf(" exec ", StringComparison.OrdinalIgnoreCase)) { string user = null, stime = null, cmd = null, snhjid = null; if (ParseExecLogLine(ln, ref user, ref stime, ref cmd, ref snhjid)) { DateTime time = DateTime.Parse(stime); if (time > lasthistory) { string secondarg = ""; { int secondargstart = 0; if (cmd.StartsWith("\"")) { secondargstart = cmd.IndexOf('"', 1); secondargstart++; } secondargstart = cmd.IndexOf(' ', secondargstart); secondargstart++; int secondargend = cmd.IndexOf(' ', secondargstart); if (-1 != secondargend) { secondarg = cmd.Substring(secondargstart, secondargend - secondargstart).Trim('"'); } } if (0 == string.Compare("exec", secondarg, StringComparison.OrdinalIgnoreCase)) { NotifyInfo.ConfigNewHistory cnh = new NotifyInfo.ConfigNewHistory(); cnh.JID = long.Parse(snhjid); //cnh.User = user; //cnh.Time = stime; //cnh.Command = cmd; cnh.History = ln; ninfo.NewHistory.Add(cnh); } } } } } } } catch (System.IO.FileNotFoundException) { } { ninfo.LastHistoryTime = starttime.ToString(); ninfo.Save(); } { // Handle the 'notifyfinish <JobInfo>' cases now. for (int inh = 0; inh < ninfo.NewHistory.Count; inh++) { NotifyInfo.ConfigNewHistory cnh = ninfo.NewHistory[inh]; if (!System.IO.File.Exists(cnh.JID.ToString() + ".jid")) { ninfo.NewHistory.RemoveAt(inh); inh--; ninfo.Save(); for (int i = 0; i < ninfo.Notify.Count; i++) { NotifyInfo.NEntry ne = ninfo.Notify[i]; if (-1 == ne.WaitOnJID) { if (System.Text.RegularExpressions.Regex.IsMatch(cnh.History, ne.WaitOnHistoryRegex, System.Text.RegularExpressions.RegexOptions.IgnoreCase)) { try { string clustername = "N/A"; string SMTP; { if (dc == null) { dc = MySpace.DataMining.AELight.dfs.ReadDfsConfig_unlocked( MySpace.DataMining.AELight.dfs.DFSXMLNAME); } if (dc.ClusterName != null) { clustername = dc.ClusterName; } SMTP = dc.SMTP; } if (null == SMTP) { throw new Exception("SMTP server is null"); } string subject = "Qizmt Notification"; string body; { StringBuilder sbbody = new StringBuilder(); sbbody.AppendLine(ne.GetFinishedUserMessage()); sbbody.AppendFormat("{1} - Cluster Name{0}", Environment.NewLine, clustername); //sbbody.AppendFormat("{1} - Command{0}", Environment.NewLine, cnh.Command); // Insecure. sbbody.AppendFormat("{1} - Job Identifier{0}", Environment.NewLine, cnh.JID); sbbody.AppendFormat("{1} - Notify Identifier{0}", Environment.NewLine, ne.ID); sbbody.AppendFormat("{1} - Completion Time{0}", Environment.NewLine, DateTime.Now); sbbody.AppendFormat("{1} - User Added Notify{0}", Environment.NewLine, ne.GetPlainUserAdded()); body = sbbody.ToString(); } SendQNEmail(SMTP, ne.Email, subject, body); } catch (Exception e) { throw new Exception("Problem sending email to " + ne.Email + " after JID " + ne.WaitOnJID + " finished (notifyfinish <JobInfo>)", e); } System.Threading.Thread.Sleep(1000 * 3); } } } wait = false; break; } } } } } for (int i = 0; i < ninfo.Notify.Count; i++) { NotifyInfo.NEntry ne = ninfo.Notify[i]; if (-1 == ne.WaitOnJID) { } else if (!System.IO.File.Exists(ne.WaitOnJID.ToString() + ".jid")) { ninfo.Notify.RemoveAt(i); i--; ninfo.Save(); try { string clustername = "N/A"; string SMTP; { if (dc == null) { dc = MySpace.DataMining.AELight.dfs.ReadDfsConfig_unlocked( MySpace.DataMining.AELight.dfs.DFSXMLNAME); } if (dc.ClusterName != null) { clustername = dc.ClusterName; } SMTP = dc.SMTP; } if (null == SMTP) { throw new Exception("SMTP server is null"); } string subject = "Qizmt Notification"; string body; { StringBuilder sbbody = new StringBuilder(); sbbody.AppendLine(ne.GetFinishedUserMessage()); sbbody.AppendFormat("{1} - Cluster Name{0}", Environment.NewLine, clustername); sbbody.AppendFormat("{1} - Job Identifier{0}", Environment.NewLine, ne.WaitOnJID); sbbody.AppendFormat("{1} - Notify Identifier{0}", Environment.NewLine, ne.ID); sbbody.AppendFormat("{1} - Completion Time{0}", Environment.NewLine, DateTime.Now); sbbody.AppendFormat("{1} - User Added Notify{0}", Environment.NewLine, ne.GetPlainUserAdded()); body = sbbody.ToString(); } SendQNEmail(SMTP, ne.Email, subject, body); } catch (Exception e) { throw new Exception("Problem sending email to " + ne.Email + " after JID " + ne.WaitOnJID + " finished (notifyfinish <JobID>)", e); } wait = false; break; } } } } }