private void DataReceived(object sender, DataReceivedEventArgs dataReceivedEventArgs) { var stdout = dataReceivedEventArgs.Data; if (stdout == null) { return; } var regex = new Regex(@"(?<currentdir>[a-zA-Z]:\\[^>]*)(>$)", RegexOptions.Multiline); var match = regex.Match(stdout); if (match.Success) { CurrentDir = match.Groups["currentdir"].ToString(); if (CurrentDir.Contains(HomeDir)) { stdout = "[" + CurrentDir.Replace(HomeDir, "~") + "] " + "(な・ω・ゆ)" + " $ "; } else { stdout = "[" + CurrentDir + "] " + "(な・ω・ゆ)" + " $ "; } } BackBuffer += stdout.Replace(CurrentDir + ">", "") + "\r\n"; }
public static void Callstack(string[] args) { if (args.Length <= 1) { Console.Error.WriteLine("Expected host"); return; } string hostcs = args[1]; string netpath = @"\\" + hostcs + @"\" + CurrentDir.Replace(':', '$'); { int protopid; string sprotopid; string protofp = netpath + @"\protocol.pid.rdbms"; for (; ; System.Threading.Thread.Sleep(1000 * 1)) { try { string protofcontent; using (System.IO.FileStream f = new System.IO.FileStream(protofp, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read | System.IO.FileShare.Write | System.IO.FileShare.Delete)) { System.IO.StreamReader sr = new System.IO.StreamReader(f); protofcontent = sr.ReadToEnd(); sr.Close(); } { int inl = protofcontent.IndexOf('\n'); // Continue if no \n. if (-1 != inl) { sprotopid = protofcontent.Substring(0, inl).Trim(); break; } } } catch (System.IO.FileNotFoundException) { Console.Error.WriteLine("No protocol process for host {0}", hostcs); return; } } protopid = int.Parse(sprotopid); sprotopid = protopid.ToString(); // Normalize. Console.WriteLine("Waiting on protocol callstack..."); { string path = netpath + @"\" + sprotopid + ".trace.rdbms"; for (; System.IO.File.Exists(path); System.Threading.Thread.Sleep(1000 * 1)) { } string tpath = "pid" + System.Diagnostics.Process.GetCurrentProcess().Id + "tracing.proto" + sprotopid + ".tof.rdbms"; System.IO.File.WriteAllText(path, tpath + Environment.NewLine + "."); } for (int tries = 0; ; tries++) { if (0 != tries) { System.Threading.Thread.Sleep(1000 * 3); } { string tpath = netpath + @"\" + "pid" + System.Diagnostics.Process.GetCurrentProcess().Id + "tracing.proto" + sprotopid + ".tof.rdbms"; { string toutput = ReadTraceFromFile(tpath); if (null == toutput) { if (!System.IO.File.Exists(protofp)) { Console.WriteLine(); Console.WriteLine("Protocol no longer running"); try { System.IO.File.Delete(netpath + @"\" + sprotopid + ".trace.rdbms"); } catch { } break; } } else { Console.WriteLine(); Console.WriteLine(toutput); try { System.IO.File.Delete(tpath); } catch { } break; } } } } Console.WriteLine(); } }
private static void ViewLog(string[] args) { int maxentries = 1000; string[] hosts = Utils.GetQizmtHosts(); if (hosts.Length == 0) { Console.Error.WriteLine("No Qizmt host is found."); return; } List <string> logpaths = new List <string>(); { string execpath = CurrentDir.Replace(':', '$'); foreach (string host in hosts) { logpaths.Add(@"\\" + host + @"\" + execpath + @"\errors.txt"); } } const int MAXBYTE = 1024 * 1024 * 64; int maxbytepart = MAXBYTE / logpaths.Count; int maxentriespart = maxentries / logpaths.Count; if (maxentries % logpaths.Count != 0) { maxentriespart++; } List <string[]> allentries = new List <string[]>(logpaths.Count); RDBMS_Admin.ThreadTools <string> .Parallel( new Action <string>( delegate(string logpath) { if (!System.IO.File.Exists(logpath)) { return; } string token = Environment.NewLine + Environment.NewLine + "["; System.IO.FileStream fs = null; try { fs = new System.IO.FileStream(logpath, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite); if (fs.Length > maxbytepart * 2) { fs.Position = fs.Length - maxbytepart; } int ib = 0; List <long> idx = new List <long>(); long entryStart = 0; while ((ib = fs.ReadByte()) > -1) { if (ib == (int)token[0]) { bool istoken = true; for (int i = 1; i < token.Length; i++) { if (fs.ReadByte() != (int)token[i]) { istoken = false; break; } } if (istoken) { idx.Add(entryStart); entryStart = fs.Position - 1; } } } //get the last entryStart. if (idx.Count > 0) { if (entryStart != idx[idx.Count - 1]) { idx.Add(entryStart); } } else { //1 entry only. if (fs.Length > 0) { idx.Add(entryStart); } } if (idx.Count == 0) { return; } long flen = fs.Length; int startidx = idx.Count > maxentriespart ? idx.Count - maxentriespart : 0; long offset = idx[startidx]; long buflen = flen - offset; while (buflen > maxbytepart && startidx < idx.Count - 1) { startidx++; offset = idx[startidx]; buflen = flen - offset; } if (buflen > maxbytepart) { throw new Exception("log too large"); } byte[] buf = new byte[buflen]; fs.Position = offset; fs.Read(buf, 0, buf.Length); fs.Close(); fs = null; string[] entries = new string[idx.Count - startidx]; for (int i = startidx; i < idx.Count; i++) { int pos = (int)(idx[i] - offset); int bytecount = 0; if (i < idx.Count - 1) { bytecount = (int)(idx[i + 1] - offset - pos); } else { bytecount = buf.Length - pos; } entries[i - startidx] = System.Text.Encoding.ASCII.GetString(buf, pos, bytecount); } lock (allentries) { allentries.Add(entries); } } catch { if (fs != null) { fs.Close(); fs = null; } throw; } } ), logpaths, logpaths.Count); if (allentries.Count == 0) { Console.Error.WriteLine("No log entries found."); return; } Console.WriteLine("-"); Console.WriteLine("Log entries:"); Console.WriteLine("-"); if (allentries.Count == 1) { foreach (string entry in allentries[0]) { Console.Write(entry); Console.WriteLine("----------------------------------------------------------------"); } Console.WriteLine("-"); Console.WriteLine("Entries displayed: {0}", allentries[0].Length); Console.WriteLine("-"); } else { List <KeyValuePair <DateTime, string> > list = new List <KeyValuePair <DateTime, string> >(); foreach (string[] entries in allentries) { foreach (string entry in entries) { int del = entry.IndexOf('M'); //AM or PM string sdate = entry.Substring(1, del); try { DateTime dt = DateTime.Parse(sdate); list.Add(new KeyValuePair <DateTime, string>(dt, entry)); } catch { } } } list.Sort(delegate(KeyValuePair <DateTime, string> x, KeyValuePair <DateTime, string> y) { return(x.Key.CompareTo(y.Key)); }); int start = list.Count > maxentries ? list.Count - maxentries : 0; for (int i = start; i < list.Count; i++) { Console.Write(list[i].Value); Console.WriteLine("----------------------------------------------------------------"); } Console.WriteLine("-"); Console.WriteLine("Entries displayed: {0}", list.Count - start); Console.WriteLine("-"); } }
private static void RepairRIndexes(string[] args) { string[] hosts = Utils.GetQizmtHosts(); if (hosts.Length == 0) { Console.Error.WriteLine("No Qizmt host is found."); return; } ConsoleColor oldcolor = Console.ForegroundColor; Console.WriteLine(); Console.WriteLine("Getting the master index..."); { string currentdir = CurrentDir.Replace(':', '$'); Dictionary <string, int> mihash = new Dictionary <string, int>(hosts.Length); foreach (string host in hosts) { string mipath = @"\\" + host + @"\" + currentdir + @"\sys.indexes"; try { string mitext = System.IO.File.ReadAllText(mipath); if (!mihash.ContainsKey(mitext)) { mihash.Add(mitext, 1); } else { mihash[mitext]++; } } catch { Console.WriteLine("Master index is skipped at:" + mipath); } } //Get master index with the highest count. System.Xml.XmlDocument mi = null; { string themi = ""; int max = Int32.MinValue; foreach (KeyValuePair <string, int> pair in mihash) { if (pair.Value > max) { max = pair.Value; themi = pair.Key; } } if (themi.Length == 0) { Console.Error.WriteLine("Cannot find the master index."); return; } else { Console.WriteLine("Master index found. Copies count: {0}", max); mi = new System.Xml.XmlDocument(); mi.LoadXml(themi); } } //Create statements List <string> skipped = new List <string>(); Dictionary <string, string> creates = null; { System.Xml.XmlNodeList indexes = mi.SelectNodes("indexes/index"); if (indexes.Count == 0) { Console.WriteLine("There is no rindex in the master index. Nothing to repair."); return; } Console.WriteLine(); Console.WriteLine(indexes.Count.ToString() + " rindexes were found from the master index."); Console.WriteLine(); Console.WriteLine("Preparing RINDEX CREATE statements..."); creates = new Dictionary <string, string>(indexes.Count); foreach (System.Xml.XmlNode index in indexes) { string iname = index["name"].InnerText; int ordinal = Int32.Parse(index["ordinal"].InnerText); bool updatememoryonly = (index["updatememoryonly"] != null && index["updatememoryonly"].InnerText == "1"); bool pin = (index["pin"].InnerText == "1"); bool pinHash = (index["pinHash"].InnerText == "1"); bool keepvalueorder = (index["keepValueOrder"].InnerText == "1"); string outliermode = index["outlier"]["mode"].InnerText; int outliermax = Int32.Parse(index["outlier"]["max"].InnerText); string tablename = index["table"]["name"].InnerText; System.Xml.XmlNodeList cols = index.SelectNodes("table/column"); if (ordinal >= cols.Count) { string msg = "Skipped: Cannot generate CREATE statement for rindex: " + iname + ". Reason: ordinal >= columns count of table; ordinal: " + ordinal.ToString() + "; columns count=" + cols.Count.ToString(); skipped.Add(msg); Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine(msg); Console.ForegroundColor = oldcolor; } else { string keycol = cols[ordinal]["name"].InnerText; string create = "CREATE RINDEX " + iname + " FROM " + tablename; if (updatememoryonly) { create += " updatememoryonly"; } if (pin & pinHash) { create += " pinmemoryhash"; } else if (pin) { create += " pinmemory"; } else if (pinHash) // should never be this case. { create += " pinmemoryhash"; } else { create += " diskonly"; } if (keepvalueorder) { create += " keepvalueorder"; } if (string.Compare(outliermode, "none", true) != 0) { create += " outlier " + outliermode + " " + outliermax.ToString(); } create += " on " + keycol; creates.Add(iname, create); } } } { string log = "rdbms_admin repairrindexes command issued:" + Environment.NewLine + "CREATE RINDEX statements prepared:" + Environment.NewLine; foreach (KeyValuePair <string, string> pair in creates) { log += pair.Value + Environment.NewLine; } LogOutputToFile(log); } Console.WriteLine(); Console.WriteLine(creates.Count.ToString() + " statements prepared."); if (skipped.Count > 0) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine(skipped.Count.ToString() + " statements skipped."); Console.ForegroundColor = oldcolor; } if (creates.Count == 0) { Console.WriteLine("All statements were skipped. Nothing to repair."); return; } //Drop all indexes Console.WriteLine(); Console.WriteLine("Dropping all rindexes..."); DeleteRIndexes(args); Console.WriteLine("Rindexes dropped."); //Create rindexes Console.WriteLine(); Console.WriteLine("Creating rindexes..."); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("Qizmt_DataProvider"); Dictionary <string, string> failed = new Dictionary <string, string>(creates.Count); int createdcount = 0; using (DbConnection conn = fact.CreateConnection()) { conn.ConnectionString = "Data Source = localhost"; foreach (KeyValuePair <string, string> pair in creates) { string iname = pair.Key; string create = pair.Value; try { Console.WriteLine("Creating rindex {0}...", iname); conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = create; cmd.ExecuteNonQuery(); conn.Close(); createdcount++; Console.WriteLine("Created successfully"); } catch (Exception e) { string msg = "Failed to create rindex: " + iname + ". Statement:" + create + ". Error:" + e.ToString(); failed.Add(iname, msg); Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine(msg); Console.ForegroundColor = oldcolor; } } } Console.WriteLine(); Console.WriteLine(createdcount.ToString() + " RIndexes were created successfully."); if (failed.Count > 0) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine(failed.Count.ToString() + " RIndexes were skipped because of error."); Console.ForegroundColor = oldcolor; } } }
private static void VerifyRIndexes(string[] args) { string[] hosts = Utils.GetQizmtHosts(); if (hosts.Length == 0) { Console.Error.WriteLine("No Qizmt host is found."); return; } bool verbose = args.Length > 1 && string.Compare(args[1], "-v", true) == 0; string currentdir = CurrentDir.Replace(':', '$'); int threadcount = hosts.Length; if (threadcount > 15) { threadcount = 15; } bool identicalsysindexes = true; { Console.WriteLine(); Console.WriteLine("Getting sysindexes state..."); if (hosts.Length > 1) { System.Xml.XmlDocument prevdoc = null; for (int i = 0; i < hosts.Length; i++) { string sysindex = @"\\" + hosts[i] + @"\" + currentdir + @"\sys.indexes"; System.Xml.XmlDocument doc = null; if (System.IO.File.Exists(sysindex)) { doc = new System.Xml.XmlDocument(); doc.Load(sysindex); } if (i > 0) { //compare if (!CompareSysIndexes(prevdoc, doc)) { identicalsysindexes = false; break; } } prevdoc = doc; } } } Dictionary <string, List <string> > leaks = new Dictionary <string, List <string> >(); Dictionary <string, Dictionary <string, int> > duprindexes = new Dictionary <string, Dictionary <string, int> >(); Dictionary <string, Dictionary <string, int> > duptables = new Dictionary <string, Dictionary <string, int> >(); Dictionary <string, Dictionary <string, int> > rindexeswithmissingfiles = new Dictionary <string, Dictionary <string, int> >(); Dictionary <string, int> okrindexesscore = new Dictionary <string, int>(); Dictionary <string, string> okrindexestypeinfo = new Dictionary <string, string>(); RDBMS_Admin.ThreadTools <string> .Parallel( new Action <string>( delegate(string host) { if (verbose) { lock (leaks) { Console.WriteLine(); Console.WriteLine("Getting info: {0}...", host); } } string root = @"\\" + host + @"\" + currentdir; string sysindex = root + @"\sys.indexes"; Dictionary <string, int> rindexes = new Dictionary <string, int>(); Dictionary <string, int> tables = new Dictionary <string, int>(); if (System.IO.File.Exists(sysindex)) { System.Xml.XmlDocument doc = new System.Xml.XmlDocument(); doc.Load(sysindex); System.Xml.XmlNodeList nodes = doc.SelectNodes("//index"); foreach (System.Xml.XmlNode node in nodes) { string rindexname = node["name"].InnerText.ToLower(); string tablename = node["table"]["name"].InnerText.ToLower(); bool pin = node["pin"].InnerText == "1"; bool ok = true; if (!rindexes.ContainsKey(rindexname)) { rindexes.Add(rindexname, 1); } else { ok = false; lock (duprindexes) { if (!duprindexes.ContainsKey(host)) { duprindexes.Add(host, new Dictionary <string, int>()); } if (!duprindexes[host].ContainsKey(rindexname)) { duprindexes[host].Add(rindexname, 0); } duprindexes[host][rindexname]++; } } if (!tables.ContainsKey(tablename)) { tables.Add(tablename, 1); } else { ok = false; lock (duptables) { if (!duptables.ContainsKey(host)) { duptables.Add(host, new Dictionary <string, int>()); } if (!duptables[host].ContainsKey(tablename)) { duptables[host].Add(tablename, 0); } duptables[host][tablename]++; } } if (!System.IO.File.Exists(root + @"\ind.Index." + rindexname + ".ind")) { ok = false; lock (rindexeswithmissingfiles) { if (!rindexeswithmissingfiles.ContainsKey(host)) { rindexeswithmissingfiles.Add(host, new Dictionary <string, int>()); } if (!rindexeswithmissingfiles[host].ContainsKey(rindexname)) { rindexeswithmissingfiles[host].Add(rindexname, 0); } rindexeswithmissingfiles[host][rindexname]++; } } if (pin && !System.IO.File.Exists(root + @"\ind.Pin." + rindexname + ".ind")) { ok = false; lock (rindexeswithmissingfiles) { if (!rindexeswithmissingfiles.ContainsKey(host)) { rindexeswithmissingfiles.Add(host, new Dictionary <string, int>()); } if (!rindexeswithmissingfiles[host].ContainsKey(rindexname)) { rindexeswithmissingfiles[host].Add(rindexname, 0); } rindexeswithmissingfiles[host][rindexname]++; } } if (ok) { lock (okrindexesscore) { if (!okrindexesscore.ContainsKey(rindexname)) { okrindexesscore.Add(rindexname, 0); } okrindexesscore[rindexname]++; } lock (okrindexestypeinfo) { if (!okrindexestypeinfo.ContainsKey(rindexname)) { okrindexestypeinfo.Add(rindexname, GetRIndexTypeInfo(node)); } } } } } string[] indfiles = System.IO.Directory.GetFiles(root, "ind.*.ind"); foreach (string indfile in indfiles) { string[] parts = indfile.Split('.'); string rindexname = parts[2].ToLower(); if (!rindexes.ContainsKey(rindexname)) { lock (leaks) { if (!leaks.ContainsKey(host)) { leaks.Add(host, new List <string>()); } leaks[host].Add(indfile); } } } } ), hosts, threadcount); if (identicalsysindexes) { Console.WriteLine(); Console.WriteLine("State of sysindexes on cluster:"); Console.WriteLine("All identical"); } Console.WriteLine(); Console.WriteLine("Healthy Hosts:"); bool unhealthyhostfound = false; foreach (string host in hosts) { if (!(leaks.ContainsKey(host) || duprindexes.ContainsKey(host) || duptables.ContainsKey(host) || rindexeswithmissingfiles.ContainsKey(host))) { Console.WriteLine(); Console.WriteLine(host); Console.WriteLine("No leak"); Console.WriteLine("No duplicated rindex"); Console.WriteLine("No duplicated table"); Console.WriteLine("No rindex with missing meta files"); } else { unhealthyhostfound = true; } } Console.WriteLine(); Console.WriteLine("Valid RIndexes:"); foreach (string key in okrindexesscore.Keys) { if (okrindexesscore[key] == hosts.Length) { Console.WriteLine(okrindexestypeinfo[key]); } } ConsoleColor oldColor = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Red; if (!identicalsysindexes) { Console.WriteLine(); Console.WriteLine("State of sysindexes on cluster:"); Console.WriteLine("Not identical"); } if (unhealthyhostfound) { Console.WriteLine(); Console.WriteLine("Unhealthy Hosts:"); foreach (string host in hosts) { if (leaks.ContainsKey(host) || duprindexes.ContainsKey(host) || duptables.ContainsKey(host) || rindexeswithmissingfiles.ContainsKey(host)) { Console.WriteLine(); Console.WriteLine(host); if (leaks.ContainsKey(host)) { Console.WriteLine("Leaks found"); if (verbose) { List <string> dups = leaks[host]; foreach (string dp in dups) { Console.WriteLine(" " + dp); } } } else { Console.WriteLine("No leak"); } if (duprindexes.ContainsKey(host)) { Console.WriteLine("Duplicated rindex found"); if (verbose) { foreach (string dp in duprindexes[host].Keys) { Console.WriteLine(" " + dp); } } } else { Console.WriteLine("No duplicated rindex"); } if (duptables.ContainsKey(host)) { Console.WriteLine("Duplicated table found"); if (verbose) { foreach (string dp in duptables[host].Keys) { Console.WriteLine(" " + dp); } } } else { Console.WriteLine("No duplicated table"); } if (rindexeswithmissingfiles.ContainsKey(host)) { Console.WriteLine("Rindex with missing meta files found"); if (verbose) { foreach (string dp in rindexeswithmissingfiles[host].Keys) { Console.WriteLine(" " + dp); } } } else { Console.WriteLine("No rindex with missing meta files"); } } } } Console.ForegroundColor = oldColor; }
private static void DeleteRIndexes(string[] args) { string[] hosts = Utils.GetQizmtHosts(); if (hosts.Length == 0) { Console.Error.WriteLine("No Qizmt host is found."); return; } const int MAX_TRIES = 10; Dictionary <string, StringBuilder> errs = new Dictionary <string, StringBuilder>(hosts.Length); string currentdir = CurrentDir.Replace(':', '$'); int threadcount = hosts.Length; if (threadcount > 15) { threadcount = 15; } RDBMS_Admin.ThreadTools <string> .Parallel( new Action <string>( delegate(string host) { int triesremain = MAX_TRIES; string root = @"\\" + host + @"\" + currentdir + @"\"; string[] indfiles = System.IO.Directory.GetFiles(root, "ind.*.ind"); foreach (string indfile in indfiles) { for (; ;) { try { System.IO.File.Delete(indfile); lock (hosts) { Console.Write('.'); } break; } catch (Exception e) { if (--triesremain <= 0) { lock (hosts) { if (!errs.ContainsKey(host)) { errs.Add(host, new StringBuilder(1024)); } StringBuilder sb = errs[host]; sb.Append(Environment.NewLine).Append("File: "). Append(indfile).Append("; Delete error: ").Append(e.ToString()); } break; } } } } string sysindex = root + "sys.indexes"; if (System.IO.File.Exists(sysindex)) { System.Xml.XmlDocument doc = new System.Xml.XmlDocument(); doc.Load(sysindex); System.Xml.XmlNodeList indexes = doc.SelectNodes("//index"); for (int i = 0; i < indexes.Count; i++) { indexes[i].ParentNode.RemoveChild(indexes[i]); } triesremain = MAX_TRIES; for (; ;) { try { doc.Save(sysindex); lock (hosts) { Console.Write('.'); } break; } catch (Exception e) { if (--triesremain <= 0) { lock (hosts) { if (!errs.ContainsKey(host)) { errs.Add(host, new StringBuilder(1024)); } StringBuilder sb = errs[host]; sb.Append(Environment.NewLine).Append("File: "). Append(sysindex).Append("; Save error: ").Append(e.ToString()); } break; } } } } } ), hosts, threadcount); Console.WriteLine(); if (errs.Count > 0) { Console.WriteLine("Errors encountered while trying to delete rindexes from these machines:"); foreach (string host in errs.Keys) { Console.WriteLine(host); Console.WriteLine(errs[host].ToString()); } } else { Console.WriteLine("Done"); } }
private static void ClearLog(string[] args) { string[] hosts = Utils.GetQizmtHosts(); if (hosts.Length == 0) { Console.Error.WriteLine("No Qizmt host is found."); return; } const int MAX_TRIES = 10; List <string> errs = new List <string>(hosts.Length); string logpath = CurrentDir.Replace(':', '$') + @"\errors.txt"; RDBMS_Admin.ThreadTools <string> .Parallel( new Action <string>( delegate(string host) { int triesremain = MAX_TRIES; string fn = @"\\" + host + @"\" + logpath; for (; ;) { try { System.IO.File.Delete(fn); lock (hosts) { Console.Write('.'); } return; } catch (Exception e) { if (--triesremain <= 0) { lock (hosts) { errs.Add(host); } break; } } } } ), hosts, hosts.Length); Console.WriteLine(); if (errs.Count > 0) { Console.WriteLine("Errors encountered while trying to clear logs from these machines:"); foreach (string e in errs) { Console.WriteLine(e); } } else { Console.WriteLine("Done"); } }