static List <Report> get_reports(DbConnection dbc) { List <Report> reports = new List <Report>(); foreach (Record crawler in dbc.Get("SELECT * FROM Crawlers WHERE State<>" + (int)Crawler.State.DISABLED).GetRecordset()) { Report report = new Report(); reports.Add(report); report.Source = (string)crawler["Id"]; report.SourceType = ReportSourceType.CRAWLER; DateTime earliest_start_time = DateTime.Now.AddSeconds(-(int)crawler["RunTimeSpan"]); Record start_message = dbc["SELECT * FROM Messages WHERE Source=@Source AND Value LIKE '" + CrawlerApi.MessageMark.STARTED + "%' ORDER BY Time DESC"].GetFirstRecord("@Source", crawler["Id"]); Crawler.SessionState state = (Crawler.SessionState)(int) dbc["SELECT _LastSessionState FROM Crawlers WHERE Id=@Id"].GetSingleValue("@Id", crawler["Id"]); if (start_message == null || (DateTime)start_message["Time"] < earliest_start_time) { if (state == Crawler.SessionState.STARTED) { report.MessageType = Log.MessageType.WARNING; report.Value = "LONG WORK"; report.Details = "Works longer than its RunTimeSpane"; continue; } report.MessageType = Log.MessageType.ERROR; report.Value = "NO START"; report.Details = "Not started within its RunTimeSpan"; continue; } Record end_message = dbc["SELECT * FROM Messages WHERE Source=@Source AND (Value LIKE '" + CrawlerApi.MessageMark.ABORTED + "%' OR Value LIKE '" + CrawlerApi.MessageMark.UNCOMPLETED + "%' OR Value LIKE '" + CrawlerApi.MessageMark.COMPLETED + "%') ORDER BY Time DESC"].GetFirstRecord("@Source", crawler["Id"]); if (end_message == null) { if (state == Crawler.SessionState.KILLED) { report.MessageType = Log.MessageType.ERROR; report.Value = "KILLED"; report.Details = "KIlled by Manager."; continue; } report.MessageType = Log.MessageType.INFORM; report.Value = "RUNNING"; report.Details = "Running"; continue; } if (Regex.IsMatch((string)end_message["Value"], @"^" + CrawlerApi.MessageMark.ABORTED)) { report.MessageType = Log.MessageType.ERROR; report.Value = "ABORTED"; report.Details = "Last session is " + CrawlerApi.MessageMark.ABORTED; continue; } if (Regex.IsMatch((string)end_message["Value"], @"^" + CrawlerApi.MessageMark.UNCOMPLETED)) { report.MessageType = Log.MessageType.WARNING; report.Value = "UNCOMPLETED"; report.Details = "Last session is " + CrawlerApi.MessageMark.UNCOMPLETED; continue; } if (Regex.IsMatch((string)end_message["Value"], @"^" + CrawlerApi.MessageMark.COMPLETED)) { report.MessageType = Log.MessageType.INFORM; report.Value = "COMPLETED"; report.Details = "Completed"; continue; } report.MessageType = Log.MessageType.ERROR; report.Value = "SYSTEM ERROR"; report.Details = "Unknown MessageMark"; } foreach (Record service in dbc.Get("SELECT * FROM Services WHERE State<>" + (int)Service.State.DISABLED).GetRecordset()) { Report report = new Report(); reports.Add(report); report.Source = (string)service["Id"]; report.SourceType = ReportSourceType.SERVICE; DateTime earliest_start_time = DateTime.Now.AddSeconds(-(int)service["RunTimeSpan"]); Record start_message = dbc["SELECT * FROM Messages WHERE Source=@Source AND Value LIKE '" + Service.MessageMark.STARTED + "%' ORDER BY Time DESC"].GetFirstRecord("@Source", service["Id"]); if (start_message == null || (DateTime)start_message["Time"] < earliest_start_time) { Service.SessionState state = (Service.SessionState)(int) dbc["SELECT _LastSessionState FROM Services WHERE Id=@Id"].GetSingleValue("@Id", service["Id"]); if (state == SessionState.STARTED) { report.MessageType = Log.MessageType.WARNING; report.Value = "LONG WORK"; report.Details = "Works longer than its RunTimeSpane"; continue; } report.MessageType = Log.MessageType.ERROR; report.Value = "NO START"; report.Details = "Not started within its RunTimeSpan"; continue; } Record end_message = dbc["SELECT * FROM Messages WHERE Source=@Source AND (Value LIKE '" + Service.MessageMark.ABORTED + "%' OR Value LIKE '" + Service.MessageMark.ERROR + "%' OR Value LIKE '" + Service.MessageMark.COMPLETED + "%') ORDER BY Time DESC"].GetFirstRecord("@Source", service["Id"]); if (end_message == null) { report.MessageType = Log.MessageType.INFORM; report.Value = "RUNNING"; report.Details = "Running"; continue; } if (Regex.IsMatch((string)end_message["Value"], @"^" + Service.MessageMark.ABORTED)) { report.MessageType = Log.MessageType.ERROR; report.Value = "ABORTED"; report.Details = "Last session is " + Service.MessageMark.ABORTED; continue; } if (Regex.IsMatch((string)end_message["Value"], @"^" + Service.MessageMark.ERROR)) { report.MessageType = Log.MessageType.ERROR; report.Value = "ERRORS"; report.Details = "Last session has errors"; continue; } if (Regex.IsMatch((string)end_message["Value"], @"^" + Service.MessageMark.COMPLETED)) { report.MessageType = Log.MessageType.INFORM; report.Value = "COMPLETED"; report.Details = "Completed"; continue; } report.MessageType = Log.MessageType.ERROR; report.Value = "SYSTEM ERROR"; report.Details = "Unknown MessageMark"; } return(reports); }
static void manage_services() { //////////////////////////////////////////////////////////// //Killing disabled service processes //////////////////////////////////////////////////////////// Recordset rs = db[@"SELECT Id, ISNULL(_LastStartTime, 0) AS _LastStartTime, ISNULL(_LastEndTime, 0) AS _LastEndTime, _LastProcessId, _LastLog, AdminEmails, _LastSessionState FROM Services WHERE _LastEndTime IS NULL AND State=" + (int)Service.State.DISABLED].GetRecordset(); foreach (Dictionary <string, object> r in rs) { string service_id = (string)r["Id"]; Process p = GetProcess((int?)r["_LastProcessId"], service_id); if (p == null) { continue; } Log.Main.Warning("Killing " + service_id + "as disabled"); p.Kill(); Thread.Sleep(2000); if (IsProcessAlive((int?)r["_LastProcessId"], service_id)) { Log.Main.Error("Could not kill " + service_id); } else { db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.KILLED + ", _LastEndTime=GETDATE() WHERE Id=@Id"].Execute("@Id", service_id); } } //////////////////////////////////////////////////////////// //Process service commands //////////////////////////////////////////////////////////// rs = db[@"SELECT Id, ISNULL(_LastStartTime, 0) AS _LastStartTime, ISNULL(_LastEndTime, 0) AS _LastEndTime, _LastProcessId, _LastLog, AdminEmails, _LastSessionState, Command FROM Services WHERE State<>" + (int)Service.State.DISABLED + " AND Command<>" + (int)Service.Command.EMPTY].GetRecordset(); foreach (Dictionary <string, object> r in rs) { string service_id = (string)r["Id"]; Process p = GetProcess((int?)r["_LastProcessId"], service_id); Service.Command command = (Service.Command)(int) r["Command"]; switch (command) { case Service.Command.RESTART: if (p == null) { db["UPDATE Services SET Command=" + (int)Service.Command.EMPTY + ", _NextStartTime=DATEADD(ss, -1, GETDATE()) WHERE Id=@Id"].Execute("@Id", service_id); break; } Log.Main.Warning("Killing " + service_id + " as marked " + command); p.Kill(); Thread.Sleep(2000); if (IsProcessAlive((int?)r["_LastProcessId"], service_id)) { db["UPDATE Services SET Command=" + (int)Service.Command.FORCE + " WHERE Id=@Id"].Execute("@Id", service_id); } else { Log.Main.Error("Could not kill " + service_id); } break; case Service.Command.STOP: if (p == null) { break; } Log.Main.Warning("Killing " + service_id + " as marked " + command); p.Kill(); Thread.Sleep(2000); if (IsProcessAlive((int?)r["_LastProcessId"], service_id)) { db["UPDATE Services SET_LastSessionState=" + (int)Service.SessionState.KILLED + ", _LastEndTime=GETDATE() WHERE Id=@Id"].Execute("@Id", service_id); } else { Log.Main.Error("Could not kill " + service_id); } break; case Service.Command.FORCE: //processed below break; case Service.Command.DISABLE_AFTER_COMPLETION: if (p == null) { db["UPDATE Services SET _LastEndTime=GETDATE(), state=" + Service.State.DISABLED + ", command" + Service.Command.EMPTY + " WHERE Id=@Id"].Execute("@Id", service_id); } break; default: throw new Exception("Service command " + command + " is not defined."); } } //////////////////////////////////////////////////////////// //Checking previously started services //////////////////////////////////////////////////////////// List <string> running_service_ids = new List <string>(); rs = db[@"SELECT DATEDIFF(ss, ISNULL(_LastStartTime, 0), GETDATE()) AS duration, Id, State, ISNULL(_LastStartTime, 0) AS _LastStartTime, ISNULL(_LastEndTime, 0) AS _LastEndTime, _LastProcessId, _LastLog, AdminEmails, _LastSessionState, RunTimeout FROM Services WHERE _LastSessionState IN (" + (int)Service.SessionState.STARTED + ", " + (int)Service.SessionState._ERROR + ", " + (int)Service.SessionState._COMPLETED + ")"].GetRecordset(); foreach (Record r in rs) { string service_id = (string)r["Id"]; string m1 = "\nStarted: " + r["_LastStartTime"] + "\nLog: " + r["_LastLog"]; int duration = (int)r["duration"]; Service.SessionState _LastSessionState = (Service.SessionState)(int) r["_LastSessionState"]; if (_LastSessionState == Service.SessionState._COMPLETED) { string m = "Service " + service_id + " completed successfully.\nTotal duration: " + (new TimeSpan(0, 0, duration)).ToString() + m1; db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.COMPLETED + " WHERE Id=@Id"].Execute("@Id", service_id); Mailer.Send(db, m, ReportSourceType.SERVICE, service_id, false); continue; } if (_LastSessionState == Service.SessionState._ERROR) { db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.ERROR + " WHERE Id=@Id"].Execute("@Id", service_id); Mailer.Send(db, "Service " + service_id + " exited with error" + m1, ReportSourceType.SERVICE, service_id); continue; } Process p = GetProcess((int?)r["_LastProcessId"], service_id); if (p == null) { db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.BROKEN + ", _NextStartTime=DATEADD(ss, RestartDelayIfBroken, GETDATE()) WHERE Id=@Id"].Execute("@Id", service_id); Mailer.Send(db, "Service " + service_id + " was broken by unknown reason" + m1, ReportSourceType.SERVICE, service_id); continue; } if (duration >= (int)r["RunTimeout"]) { Mailer.Send(db, "Service " + service_id + " is running " + (new TimeSpan(0, 0, duration)).ToString() + " seconds. It will be killed." + m1, ReportSourceType.SERVICE, service_id); p = GetProcess((int?)r["_LastProcessId"], service_id); Log.Main.Warning("Killing " + service_id); p.Kill(); Thread.Sleep(2000); if (!IsProcessAlive((int?)r["_LastProcessId"], service_id)) { db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.KILLED + ", _NextStartTime=DATEADD(ss, RestartDelayIfBroken, GETDATE()), _LastEndTime=GETDATE() WHERE Id=@Id"].Execute("@Id", service_id); } else { Log.Main.Error("Could not kill " + service_id); } continue; } running_service_ids.Add(service_id); } //////////////////////////////////////////////////////////// //Starting services //////////////////////////////////////////////////////////// rs = db[@"SELECT Id, State, Command, AdminEmails,_LastProcessId,ExeFolder FROM Services WHERE (State<>" + (int)Service.State.DISABLED + " AND GETDATE()>=_NextStartTime AND Command<>" + (int)Service.Command.STOP + @") OR Command=" + (int)Service.Command.FORCE + " ORDER BY Command, _NextStartTime"].GetRecordset(); foreach (Record r in rs) { string service_id = (string)r["Id"]; Process p = GetProcess((int?)r["_LastProcessId"], service_id); if ((int)r["Command"] == (int)Service.Command.FORCE) { Log.Main.Write("Forcing " + service_id); if ((int)r["State"] == (int)Service.State.DISABLED) { Log.Main.Error(service_id + " is disabled."); continue; } if (p != null) { Log.Main.Warning(service_id + " is running already."); db["UPDATE Services SET Command=" + (int)Service.Command.EMPTY + " WHERE Id=@Id"].Execute("@Id", service_id); continue; } if (launch_service(r, running_service_ids)) { db["UPDATE Services SET Command=" + (int)Service.Command.EMPTY + " WHERE Id=@Id"].Execute("@Id", service_id); } continue; } if (p != null) { continue; } if (running_service_ids.Count >= Properties.Settings.Default.ServiceProcessMaxNumber) { continue; } launch_service(r, running_service_ids); } }