static List <Report> get_reports(DbConnection dbc)
{
List <Report> reports = new List <Report>();
foreach (Record crawler in dbc.Get("SELECT * FROM Crawlers WHERE State<>" + (int)Crawler.State.DISABLED).GetRecordset())
{
Report report = new Report();
reports.Add(report);
report.Source = (string)crawler["Id"];
report.SourceType = ReportSourceType.CRAWLER;
DateTime earliest_start_time = DateTime.Now.AddSeconds(-(int)crawler["RunTimeSpan"]);
Record start_message = dbc["SELECT * FROM Messages WHERE Source=@Source AND Value LIKE '" + CrawlerApi.MessageMark.STARTED + "%' ORDER BY Time DESC"].GetFirstRecord("@Source", crawler["Id"]);
Crawler.SessionState state = (Crawler.SessionState)(int) dbc["SELECT _LastSessionState FROM Crawlers WHERE Id=@Id"].GetSingleValue("@Id", crawler["Id"]);
if (start_message == null || (DateTime)start_message["Time"] < earliest_start_time)
{
if (state == Crawler.SessionState.STARTED)
{
report.MessageType = Log.MessageType.WARNING;
report.Value = "LONG WORK";
report.Details = "Works longer than its RunTimeSpane";
continue;
}
report.MessageType = Log.MessageType.ERROR;
report.Value = "NO START";
report.Details = "Not started within its RunTimeSpan";
continue;
}
Record end_message = dbc["SELECT * FROM Messages WHERE Source=@Source AND (Value LIKE '" + CrawlerApi.MessageMark.ABORTED + "%' OR Value LIKE '" + CrawlerApi.MessageMark.UNCOMPLETED + "%' OR Value LIKE '" + CrawlerApi.MessageMark.COMPLETED + "%') ORDER BY Time DESC"].GetFirstRecord("@Source", crawler["Id"]);
if (end_message == null)
{
if (state == Crawler.SessionState.KILLED)
{
report.MessageType = Log.MessageType.ERROR;
report.Value = "KILLED";
report.Details = "KIlled by Manager.";
continue;
}
report.MessageType = Log.MessageType.INFORM;
report.Value = "RUNNING";
report.Details = "Running";
continue;
}
if (Regex.IsMatch((string)end_message["Value"], @"^" + CrawlerApi.MessageMark.ABORTED))
{
report.MessageType = Log.MessageType.ERROR;
report.Value = "ABORTED";
report.Details = "Last session is " + CrawlerApi.MessageMark.ABORTED;
continue;
}
if (Regex.IsMatch((string)end_message["Value"], @"^" + CrawlerApi.MessageMark.UNCOMPLETED))
{
report.MessageType = Log.MessageType.WARNING;
report.Value = "UNCOMPLETED";
report.Details = "Last session is " + CrawlerApi.MessageMark.UNCOMPLETED;
continue;
}
if (Regex.IsMatch((string)end_message["Value"], @"^" + CrawlerApi.MessageMark.COMPLETED))
{
report.MessageType = Log.MessageType.INFORM;
report.Value = "COMPLETED";
report.Details = "Completed";
continue;
}
report.MessageType = Log.MessageType.ERROR;
report.Value = "SYSTEM ERROR";
report.Details = "Unknown MessageMark";
}
foreach (Record service in dbc.Get("SELECT * FROM Services WHERE State<>" + (int)Service.State.DISABLED).GetRecordset())
{
Report report = new Report();
reports.Add(report);
report.Source = (string)service["Id"];
report.SourceType = ReportSourceType.SERVICE;
DateTime earliest_start_time = DateTime.Now.AddSeconds(-(int)service["RunTimeSpan"]);
Record start_message = dbc["SELECT * FROM Messages WHERE Source=@Source AND Value LIKE '" + Service.MessageMark.STARTED + "%' ORDER BY Time DESC"].GetFirstRecord("@Source", service["Id"]);
if (start_message == null || (DateTime)start_message["Time"] < earliest_start_time)
{
Service.SessionState state = (Service.SessionState)(int) dbc["SELECT _LastSessionState FROM Services WHERE Id=@Id"].GetSingleValue("@Id", service["Id"]);
if (state == SessionState.STARTED)
{
report.MessageType = Log.MessageType.WARNING;
report.Value = "LONG WORK";
report.Details = "Works longer than its RunTimeSpane";
continue;
}
report.MessageType = Log.MessageType.ERROR;
report.Value = "NO START";
report.Details = "Not started within its RunTimeSpan";
continue;
}
Record end_message = dbc["SELECT * FROM Messages WHERE Source=@Source AND (Value LIKE '" + Service.MessageMark.ABORTED + "%' OR Value LIKE '" + Service.MessageMark.ERROR + "%' OR Value LIKE '" + Service.MessageMark.COMPLETED + "%') ORDER BY Time DESC"].GetFirstRecord("@Source", service["Id"]);
if (end_message == null)
{
report.MessageType = Log.MessageType.INFORM;
report.Value = "RUNNING";
report.Details = "Running";
continue;
}
if (Regex.IsMatch((string)end_message["Value"], @"^" + Service.MessageMark.ABORTED))
{
report.MessageType = Log.MessageType.ERROR;
report.Value = "ABORTED";
report.Details = "Last session is " + Service.MessageMark.ABORTED;
continue;
}
if (Regex.IsMatch((string)end_message["Value"], @"^" + Service.MessageMark.ERROR))
{
report.MessageType = Log.MessageType.ERROR;
report.Value = "ERRORS";
report.Details = "Last session has errors";
continue;
}
if (Regex.IsMatch((string)end_message["Value"], @"^" + Service.MessageMark.COMPLETED))
{
report.MessageType = Log.MessageType.INFORM;
report.Value = "COMPLETED";
report.Details = "Completed";
continue;
}
report.MessageType = Log.MessageType.ERROR;
report.Value = "SYSTEM ERROR";
report.Details = "Unknown MessageMark";
}
return(reports);
}
static void manage_services()
{
////////////////////////////////////////////////////////////
//Killing disabled service processes
////////////////////////////////////////////////////////////
Recordset rs = db[@"SELECT Id,
ISNULL(_LastStartTime, 0) AS _LastStartTime, ISNULL(_LastEndTime, 0) AS _LastEndTime,
_LastProcessId, _LastLog, AdminEmails, _LastSessionState
FROM Services
WHERE _LastEndTime IS NULL AND State=" + (int)Service.State.DISABLED].GetRecordset();
foreach (Dictionary <string, object> r in rs)
{
string service_id = (string)r["Id"];
Process p = GetProcess((int?)r["_LastProcessId"], service_id);
if (p == null)
{
continue;
}
Log.Main.Warning("Killing " + service_id + "as disabled");
p.Kill();
Thread.Sleep(2000);
if (IsProcessAlive((int?)r["_LastProcessId"], service_id))
{
Log.Main.Error("Could not kill " + service_id);
}
else
{
db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.KILLED + ", _LastEndTime=GETDATE() WHERE Id=@Id"].Execute("@Id", service_id);
}
}
////////////////////////////////////////////////////////////
//Process service commands
////////////////////////////////////////////////////////////
rs = db[@"SELECT Id,
ISNULL(_LastStartTime, 0) AS _LastStartTime, ISNULL(_LastEndTime, 0) AS _LastEndTime,
_LastProcessId, _LastLog, AdminEmails, _LastSessionState, Command
FROM Services
WHERE State<>" + (int)Service.State.DISABLED + " AND Command<>" + (int)Service.Command.EMPTY].GetRecordset();
foreach (Dictionary <string, object> r in rs)
{
string service_id = (string)r["Id"];
Process p = GetProcess((int?)r["_LastProcessId"], service_id);
Service.Command command = (Service.Command)(int) r["Command"];
switch (command)
{
case Service.Command.RESTART:
if (p == null)
{
db["UPDATE Services SET Command=" + (int)Service.Command.EMPTY + ", _NextStartTime=DATEADD(ss, -1, GETDATE()) WHERE Id=@Id"].Execute("@Id", service_id);
break;
}
Log.Main.Warning("Killing " + service_id + " as marked " + command);
p.Kill();
Thread.Sleep(2000);
if (IsProcessAlive((int?)r["_LastProcessId"], service_id))
{
db["UPDATE Services SET Command=" + (int)Service.Command.FORCE + " WHERE Id=@Id"].Execute("@Id", service_id);
}
else
{
Log.Main.Error("Could not kill " + service_id);
}
break;
case Service.Command.STOP:
if (p == null)
{
break;
}
Log.Main.Warning("Killing " + service_id + " as marked " + command);
p.Kill();
Thread.Sleep(2000);
if (IsProcessAlive((int?)r["_LastProcessId"], service_id))
{
db["UPDATE Services SET_LastSessionState=" + (int)Service.SessionState.KILLED + ", _LastEndTime=GETDATE() WHERE Id=@Id"].Execute("@Id", service_id);
}
else
{
Log.Main.Error("Could not kill " + service_id);
}
break;
case Service.Command.FORCE:
//processed below
break;
case Service.Command.DISABLE_AFTER_COMPLETION:
if (p == null)
{
db["UPDATE Services SET _LastEndTime=GETDATE(), state=" + Service.State.DISABLED + ", command" + Service.Command.EMPTY + " WHERE Id=@Id"].Execute("@Id", service_id);
}
break;
default:
throw new Exception("Service command " + command + " is not defined.");
}
}
////////////////////////////////////////////////////////////
//Checking previously started services
////////////////////////////////////////////////////////////
List <string> running_service_ids = new List <string>();
rs = db[@"SELECT DATEDIFF(ss, ISNULL(_LastStartTime, 0), GETDATE()) AS duration, Id, State,
ISNULL(_LastStartTime, 0) AS _LastStartTime, ISNULL(_LastEndTime, 0) AS _LastEndTime,
_LastProcessId, _LastLog, AdminEmails, _LastSessionState, RunTimeout
FROM Services
WHERE _LastSessionState IN (" + (int)Service.SessionState.STARTED + ", " + (int)Service.SessionState._ERROR + ", " + (int)Service.SessionState._COMPLETED + ")"].GetRecordset();
foreach (Record r in rs)
{
string service_id = (string)r["Id"];
string m1 = "\nStarted: " + r["_LastStartTime"] + "\nLog: " + r["_LastLog"];
int duration = (int)r["duration"];
Service.SessionState _LastSessionState = (Service.SessionState)(int) r["_LastSessionState"];
if (_LastSessionState == Service.SessionState._COMPLETED)
{
string m = "Service " + service_id + " completed successfully.\nTotal duration: " + (new TimeSpan(0, 0, duration)).ToString() + m1;
db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.COMPLETED + " WHERE Id=@Id"].Execute("@Id", service_id);
Mailer.Send(db, m, ReportSourceType.SERVICE, service_id, false);
continue;
}
if (_LastSessionState == Service.SessionState._ERROR)
{
db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.ERROR + " WHERE Id=@Id"].Execute("@Id", service_id);
Mailer.Send(db, "Service " + service_id + " exited with error" + m1, ReportSourceType.SERVICE, service_id);
continue;
}
Process p = GetProcess((int?)r["_LastProcessId"], service_id);
if (p == null)
{
db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.BROKEN + ", _NextStartTime=DATEADD(ss, RestartDelayIfBroken, GETDATE()) WHERE Id=@Id"].Execute("@Id", service_id);
Mailer.Send(db, "Service " + service_id + " was broken by unknown reason" + m1, ReportSourceType.SERVICE, service_id);
continue;
}
if (duration >= (int)r["RunTimeout"])
{
Mailer.Send(db, "Service " + service_id + " is running " + (new TimeSpan(0, 0, duration)).ToString() + " seconds. It will be killed." + m1, ReportSourceType.SERVICE, service_id);
p = GetProcess((int?)r["_LastProcessId"], service_id);
Log.Main.Warning("Killing " + service_id);
p.Kill();
Thread.Sleep(2000);
if (!IsProcessAlive((int?)r["_LastProcessId"], service_id))
{
db["UPDATE Services SET _LastSessionState=" + (int)Service.SessionState.KILLED + ", _NextStartTime=DATEADD(ss, RestartDelayIfBroken, GETDATE()), _LastEndTime=GETDATE() WHERE Id=@Id"].Execute("@Id", service_id);
}
else
{
Log.Main.Error("Could not kill " + service_id);
}
continue;
}
running_service_ids.Add(service_id);
}
////////////////////////////////////////////////////////////
//Starting services
////////////////////////////////////////////////////////////
rs = db[@"SELECT Id, State, Command, AdminEmails,_LastProcessId,ExeFolder FROM Services
WHERE (State<>" + (int)Service.State.DISABLED + " AND GETDATE()>=_NextStartTime AND Command<>" + (int)Service.Command.STOP + @")
OR Command=" + (int)Service.Command.FORCE + " ORDER BY Command, _NextStartTime"].GetRecordset();
foreach (Record r in rs)
{
string service_id = (string)r["Id"];
Process p = GetProcess((int?)r["_LastProcessId"], service_id);
if ((int)r["Command"] == (int)Service.Command.FORCE)
{
Log.Main.Write("Forcing " + service_id);
if ((int)r["State"] == (int)Service.State.DISABLED)
{
Log.Main.Error(service_id + " is disabled.");
continue;
}
if (p != null)
{
Log.Main.Warning(service_id + " is running already.");
db["UPDATE Services SET Command=" + (int)Service.Command.EMPTY + " WHERE Id=@Id"].Execute("@Id", service_id);
continue;
}
if (launch_service(r, running_service_ids))
{
db["UPDATE Services SET Command=" + (int)Service.Command.EMPTY + " WHERE Id=@Id"].Execute("@Id", service_id);
}
continue;
}
if (p != null)
{
continue;
}
if (running_service_ids.Count >= Properties.Settings.Default.ServiceProcessMaxNumber)
{
continue;
}
launch_service(r, running_service_ids);
}
}
