public static bool RemoveJob(string jobName, string comment, string samayDB_ConnectionString) { Engine engine = GetSamayConfig(samayDB_ConnectionString); Job jobToRemove = null; foreach (var j in engine.Jobs) { if (j.JobName.ToLower() == jobName.ToLower()) { jobToRemove = j; break; } } if (jobToRemove != null) { engine.Jobs.Remove(jobToRemove); SaveSamayConfig(engine, comment, samayDB_ConnectionString); SamayLogger.Log("Successfully removed new Job '" + jobName + "'", SamayLogger.SamayEngineLogJobName, "Config", "Engine", LogLevel.Info); return(true); } SamayLogger.Log("Failed to removed new Job '" + jobName + "'. Job does not exist", SamayLogger.SamayEngineLogJobName, "Config", "Engine", LogLevel.Error); throw new Exception("Job '" + jobName + "' does not exist"); }
public SamayEngine() { //when starting from scratch Config.InitSamayDB(EnsureFullPath(Settings.Default.DatabasePath), GetDBConnectionString()); SamayLogger.InitSamayLogger(GetDBConnectionString()); SamayLogger.SetLogLevel("Startup", LogLevel.Trace); SamayLogger.SetLogLevel("Engine Stop", LogLevel.Trace); SamayLogger.SetLogLevel("Engine", LogLevel.Trace); cm = new Communication(); //open the communication channel }
public static bool SaveSamayConfig(Engine config, string comment, string samayDB_ConnectionString) { using (SQLiteConnection cn = new SQLiteConnection(samayDB_ConnectionString)) { try { cn.Open(); SQLiteCommand sCommand1 = new SQLiteCommand(@"create table if not exists ConfigHistory (ID integer primary key autoincrement, TimeStamp datetime default current_timestamp, UserID integer,Title varchar,Config varchar);", cn); sCommand1.ExecuteScalar(); SQLiteCommand sCommand2 = new SQLiteCommand(@"INSERT INTO ConfigHistory ([TimeStamp], UserID, Title, Config) VALUES (@TimeStamp, @UserID, @Title, @Config); SELECT last_insert_rowid();", cn); sCommand2.Parameters.AddRange(new[] { new SQLiteParameter { DbType = DbType.DateTime, ParameterName = "TimeStamp", Value = DateTime.UtcNow }, new SQLiteParameter { DbType = DbType.Int16, ParameterName = "UserID", Value = 1 }, new SQLiteParameter { DbType = DbType.String, ParameterName = "Title", Value = comment }, new SQLiteParameter { DbType = DbType.String, ParameterName = "Config", Value = JsonConvert.SerializeObject(config) }, }); sCommand2.ExecuteScalar(); SQLiteCommand sCommand3 = new SQLiteCommand(@"create table if not exists LOGS (ID integer primary key autoincrement, TimeStamp datetime default current_timestamp, Level varchar, Message varchar, JobName varchar, TaskName varchar);", cn); sCommand3.ExecuteScalar(); return(true); } catch (Exception ex) { SamayLogger.Log("Error while saving config. " + ex.ToString(), SamayLogger.SamayEngineLogJobName, "Config", "Startup", LogLevel.Error, DateTime.Now); return(false); } finally { cn.Close(); } } }
internal Dictionary <string, string> Initialize() { Dictionary <string, string> TaskAssemblyDict = new Dictionary <string, string>(); string taskDir = Technisient.SamayEngine.EnsureFullPath(Settings.Default.TasksDirectory); if (!Directory.Exists(taskDir)) { Directory.CreateDirectory(taskDir); } //cache all Tasks assembly names in the beginning. DirectoryInfo dI = new DirectoryInfo(taskDir); FileInfo[] files = dI.GetFiles("*.dll"); AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(CurrentDomain_AssemblyResolve); foreach (FileInfo f in files) { try { Assembly assembly = System.Reflection.Assembly.LoadFrom(f.FullName); foreach (Type classType in assembly.GetTypes()) { if (!classType.IsClass) { continue; } if (classType.BaseType.FullName == "Technisient.TaskBase") { TaskAssemblyDict.Add(classType.FullName, f.FullName); } } } catch (Exception ex) { SamayLogger.LogError("Unable to Load Assembly: " + f.FullName + "\n" + ex.ToString(), SamayLogger.SamayEngineLogJobName, "Engine", SamayLogger.SamayEngineLoggingGUID, DateTime.Now); } } AppDomain.CurrentDomain.AssemblyResolve -= new ResolveEventHandler(CurrentDomain_AssemblyResolve); return(TaskAssemblyDict); }
internal static TaskBase CreateInstanceInAppDomain(string className, AppDomain jobDomain) { //try multiple times before giving up! for (int i = 0; i < 4; i++) { try { return((TaskBase)Activator.CreateInstanceFrom(jobDomain, _TaskAssemblyDict[className], className).Unwrap()); } catch (Exception ex) { SamayLogger.LogWarning("Unable to create instance of " + className + " on Try " + i.ToString() + " Trying Again" + "\nERROR:" + ex.ToString(), SamayLogger.SamayEngineLogJobName, "Engine", SamayLogger.SamayEngineLoggingGUID, DateTime.Now); System.Threading.Thread.Sleep(500); } } return((TaskBase)Activator.CreateInstanceFrom(jobDomain, _TaskAssemblyDict[className], className).Unwrap()); }
public static bool AddJob(string jobString, string comment, string samayDB_ConnectionString) { Engine engine = GetSamayConfig(samayDB_ConnectionString); Job job = JsonConvert.DeserializeObject <Job>(jobString); foreach (var j in engine.Jobs) { if (j.JobName.ToLower() == job.JobName.ToLower()) { SamayLogger.Log("Failed to add new Job '" + job.JobName + "'. Job already exists", SamayLogger.SamayEngineLogJobName, "Config", "Engine", LogLevel.Error); throw new Exception("Job with the name '" + job.JobName + "' already exists"); } } engine.Jobs.Add(job); SaveSamayConfig(engine, comment, samayDB_ConnectionString); SamayLogger.Log("Successfully added new Job '" + job.JobName + "'", SamayLogger.SamayEngineLogJobName, "Config", "Engine", LogLevel.Info); return(true); }
public static Engine GetSamayConfig(string samayDB_ConnectionString) { // XmlSerializer serializer = new XmlSerializer(typeof(Engine)); Engine output; using (SQLiteConnection cn = new SQLiteConnection(samayDB_ConnectionString)) { try { cn.Open(); SQLiteCommand sCommand = new SQLiteCommand( @"SELECT Config FROM ConfigHistory ORDER BY TimeStamp DESC LIMIT 1", cn); string engineString = (string)sCommand.ExecuteScalar(); output = JsonConvert.DeserializeObject <Engine>(engineString); if (output == null) { throw new Exception(); } } catch (Exception ex) { SamayLogger.Log("Failed to Read Samay Engine Configuration. " + ex.ToString(), "Config", "Config Reader", "Startup", LogLevel.Info, DateTime.Now); throw ex; } finally { cn.Close(); } } //using (StringReader reader = new StringReader(File.ReadAllText("EngineConfig.xml"))) //{ // output = (Engine)serializer.Deserialize(reader); //} SamayLogger.Log("Samay Engine Configuration sucessfully read into memory", SamayLogger.SamayEngineLogJobName, "Config", "Startup", LogLevel.Info, DateTime.Now); return(output); }
private void Initialize(Engine engine) { try { SamayLogger.SetLogLevel(SamayLogger.SamayEngineLoggingGUID, engine.EngineConfig.LogLevel); LogInfo(JsonConvert.SerializeObject(engine, Formatting.Indented)); AppDomain ad = AppDomain.CreateDomain("SamayEnginedomain" + DateTime.Now.ToString()); TaskFactory tf = (TaskFactory)ad.CreateInstanceAndUnwrap( Assembly.GetExecutingAssembly().FullName, "Technisient.TaskFactory"); _TaskAssemblyDict = tf.Initialize(); AppDomain.Unload(ad); //TaskFactory.Initialize(); //at the Job level, we still use the Engine log level foreach (SamayConfig.Job job in engine.Jobs) { SamayLogger.SetLogLevel(job.Id, engine.EngineConfig.LogLevel); if ((job.TaskChain.Task == null) || (job.TaskChain.Task.Count() == 0)) { continue; } foreach (SamayConfig.Task task in job.TaskChain.Task) { SamayLogger.SetLogLevel(task.Id, task.LogLevel); } } //cache Global Exclude Dates _GlobalExcludeDates = new List <DateTime>(); if (engine.EngineConfig.GlobalExcludeDates != null) { foreach (GlobalExcludeDate dt in engine.EngineConfig.GlobalExcludeDates) { _GlobalExcludeDates.Add(dt.Date); } } _jobExcludeDates = new Dictionary <string, List <DateTime> >(); //cache JobExclude Dates foreach (Job job in engine.Jobs) { List <DateTime> dtList = new List <DateTime>(); if (job.ExcludeDates != null) { foreach (JobExcludeDates exc in job.ExcludeDates) { dtList.Add(exc.Date); } } _jobExcludeDates.Add(job.JobName, dtList); // set endtime to really high number for jobs which do not run continously if (job.ExecuteTimes != -1) { if (job.Schedules != null) { foreach (JobSchedule jobSchedule in job.Schedules) { Time t = new Time(); switch (jobSchedule.ScheduleConfig.GetType().Name) { case "EngineJobScheduleDaily": JobScheduleDaily daily = (JobScheduleDaily)jobSchedule.ScheduleConfig; t = daily.Time; break; case "EngineJobScheduleWeekly": JobScheduleWeekly weekly = (JobScheduleWeekly)jobSchedule.ScheduleConfig; t = weekly.Day.Time; break; case "EngineJobScheduleMonthly": JobScheduleMonthly monthly = (JobScheduleMonthly)jobSchedule.ScheduleConfig; t = monthly.Day.Time; break; case "EngineJobScheduleYearly": JobScheduleYearly yearly = (JobScheduleYearly)jobSchedule.ScheduleConfig; t = yearly.Month.Time; break; case "EngineJobScheduleOneTimeOnly": JobScheduleOneTimeOnly oneTime = (JobScheduleOneTimeOnly)jobSchedule.ScheduleConfig; if (!oneTime.EndDateTimeSpecified) { oneTime.EndDateTime = oneTime.StartDateTime.AddYears(10); oneTime.EndDateTimeSpecified = true; } continue; default: throw new Exception("Unknown Schedule Type: " + jobSchedule.ScheduleConfig.GetType().Name); } if (!t.EndTimeSpecified) { if (job.ExecuteTimes == 1) { t.EndTime = t.StartTime.AddMinutes(1); } else if (job.Interval.ClockTime != null) { t.EndTime = t.StartTime; int count = 0; while (count < job.ExecuteTimes) { if (job.Interval.ClockTime.Contains(t.EndTime.Minute)) { count++; } t.EndTime = t.EndTime.AddMinutes(1); } } else { int interval = (int)job.Interval.Interval_msec; t.EndTime = t.StartTime.AddMilliseconds(interval * job.ExecuteTimes).AddMinutes(1); } t.EndTimeSpecified = true; } } } } } LogInfo("Starting Up Engine"); } catch (Exception ex) { string sSource = "Technisient Samay"; string sLog = "Samay Engine"; if (!EventLog.SourceExists(sSource)) { EventLog.CreateEventSource(sSource, sLog); } EventLog.WriteEntry(sSource, ex.ToString()); } }
public void RunEngine(Time testTime, Engine engine) { _engineStatus = EngineStatus.Starting; Initialize(engine); _tasksFolder = Technisient.SamayEngine.EnsureFullPath(Settings.Default.TasksDirectory); HashSet<string> noTaskJobs = new HashSet<string>(); if (!engine.EngineConfig.Enabled) { LogWarning("Engine Has Been Disabled In The Configuration Settings. Engine Stopped."); _engineStatus = EngineStatus.Disabled; while (true) { System.Threading.Thread.Sleep(1000); //simply do nothing.. keep things running so that website can report current status } } _engineStatus = EngineStatus.Running; DateTime CURRENT_DATETIME = DateTime.Now; if (testTime != null) { CURRENT_DATETIME = testTime.StartTime; TASKS_IN_APPDOMAINS = false;//app domain dont work well with test } Thread.CurrentThread.Priority = ThreadPriority.Highest; while (true) { System.Threading.Thread.Sleep(new TimeSpan(engine.EngineConfig.SleepTicks));//ticks - 1 tick = 100 nanoseconds & 10,000 ticks = 1 msec // done all the testing and 1 millisecond works perfectly fine without using much CPU when idle on Core2Duo machine try { #region Handle Test Scenario if (testTime != null) { CURRENT_DATETIME = CURRENT_DATETIME.AddSeconds(30); if (CURRENT_DATETIME.TimeOfDay == new TimeSpan(0)) System.Diagnostics.Trace.WriteLine("Time: " + CURRENT_DATETIME.ToString()); if (CURRENT_DATETIME > testTime.EndTime) return; //LogInfo("Current Time is: " + CURRENT_DATETIME.ToString()); //System.Threading.Thread.Sleep(200); } else { CURRENT_DATETIME = DateTime.Now;//keep it constant for current run } #endregion Handle Test Scenario #region Reload Config if (_reloadConfig) { SamayLogger.Flush(); _reloadConfig = false; //make all vars local RunEngine(); //current while loop will stop forever } #endregion Reload Config #region Stop Engine if (_stopEngine) { // SamayLogger.Flush(); // bool nothingRunning = true; // TODO: allow for safe stopping. let jobs complete. //foreach (var j in _currentlyRunningJobs.Keys) //{ // if ((_currentlyRunningJobs[j].task.Status == TaskStatus.Running) || // (_currentlyRunningJobs[j].task.Status == TaskStatus.WaitingForChildrenToComplete)) // { // nothingRunning = false; // break; // } //} //if (nothingRunning) ////{ //// _engineStatus = EngineStatus.Stopped; //// LogInfo("Engine Stopped"); //// return; ////} //break; } #endregion Stop Engine #region Run System Jobs if (testTime == null) if (!_SystemJobsLastRunDate.Contains(CURRENT_DATETIME.Date)) { _SystemJobsLastRunDate.Add(CURRENT_DATETIME.Date); RunSystemJobs(engine); } #endregion Run System Jobs //Reload config every 5 minutes? NO! //whatever was started will keep running as is foreach (Job job in engine.Jobs) { lock (_currentlyRunningJobs) { if (_stopEngine) break; if (!job.Enabled) continue; if ((job.TaskChain.Task == null) || (job.TaskChain.Task.Count() == 0)) { if (!noTaskJobs.Contains(job.Id)) { //log just once LogError("Job: '" + job.JobName + "' has no Tasks defined to execute!"); noTaskJobs.Add(job.Id); } continue; } if ((_jobExcludeDates.ContainsKey(job.JobName)) && (_jobExcludeDates[job.JobName].Contains(DateTime.Today))) { continue; } if (job.DoNotRunOnGlobalExcludeDates) { if (_GlobalExcludeDates.Contains(DateTime.Today)) continue; } if (job.Schedules != null) foreach (JobSchedule schedule in job.Schedules) { #region Should Task be Run Now? if (!IsJobScheduledToRunNow(schedule, CURRENT_DATETIME)) { if (_currentlyRunningJobs.ContainsKey(job.JobName + schedule.ID)) { RunningJob temp = new RunningJob(); _currentlyRunningJobs.TryRemove(job.JobName + schedule.ID, out temp); } continue; } // Job has passed all eligibility test! Run it now :) if (!_currentlyRunningJobs.ContainsKey(job.JobName + schedule.ID)) { //This is the very first time this job is going to run in current schedule RunningJob rj = new RunningJob(); rj.firstRun = CURRENT_DATETIME; _currentlyRunningJobs.TryAdd(job.JobName + schedule.ID, rj); } RunningJob runningJob = _currentlyRunningJobs[job.JobName + schedule.ID]; if (runningJob.complete) continue;//cant remove it from _currentlyRunningJobs as it will be picked up on the next iteration! #if ALLOW_CONCURRENT if (!job.AllowConcurrent) if (runningJob.jobChainBusy) continue; #endif if ((runningJob.count >= job.ExecuteTimes) && (job.ExecuteTimes != -1)) //-1 means keep running always { SamayLogger.LogDebug("Ending to Run Job: " + job.JobName + " Ran: " + runningJob.count.ToString() + " times", job.JobName, "Engine", job.Id, CURRENT_DATETIME); runningJob.complete = true; continue; } #endregion Should Task be Run Now? bool runNow = false; //if (!(DateTime.Now.Subtract(_currentlyRunningJobs[job.JobName].lastRun).TotalMilliseconds < job.ThrottlingDelay)) if (job.ExecuteTimes == 1) { runNow = true; } else if (job.Interval.ClockTime != null) { //on the Clock if (runningJob.lastRun.ToString("g") != CURRENT_DATETIME.ToString("g")) { if (job.Interval.ClockTime.Contains(CURRENT_DATETIME.Minute)) { runNow = true; } } } else { //frequency , not on clock int interval_msec = (int)job.Interval.Interval_msec; if ((CURRENT_DATETIME.Subtract(runningJob.firstRun).TotalMilliseconds >= interval_msec * runningJob.count)) //this will try and catchup { runNow = true; } } if (runNow) { // Console.WriteLine("AppDomain: " + AppDomain.CurrentDomain.FriendlyName); runningJob.jobGUID = Guid.NewGuid(); runningJob.count++; runningJob.lastRun = CURRENT_DATETIME; SamayLogger.LogInfo("Executing Job: " + job.JobName + " Count: " + runningJob.count.ToString(), job.JobName, "Engine", job.Id, CURRENT_DATETIME); #if ALLOW_CONCURRENT _currentlyRunningJobs[job.JobName].jobChainBusy = true; #endif Job jobClone = JsonConvert.DeserializeObject<Job>(JsonConvert.SerializeObject(job), new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All, ObjectCreationHandling = ObjectCreationHandling.Reuse }); //easy cloning // SamayLogger.Log("GUID of Task: " + jobClone.Id, job.JobName, ); _currentlyRunningJobs[job.JobName + schedule.ID].task = ExecuteJob(jobClone, schedule, 0); } } } } } catch (Exception ex) { LogError(ex.ToString()); } } #if USETASKS //if (t != null) // t.Wait #endif // _engineStatus = EngineStatus.Stopped; }
private System.Threading.Tasks.Task ExecuteJob(Job job, JobSchedule schedule, long runCount) { //FUTURE: Spawn each Job out in a seperate exe?! //FUTURE: Send message to Tasks when stopped is recieve so that they can wrap up if they can TaskContext tc = new TaskContext(); tc.jobName = job.JobName; tc.runCount = _currentlyRunningJobs[job.JobName + schedule.ID].count; tc.jobGUId = _currentlyRunningJobs[job.JobName + schedule.ID].jobGUID.ToString(); tc.jobId = job.Id; //PLEASE ALWAYS MAKE IT EASY TO TURN OFF TASKS by using #defines as shown below #if USETASKS System.Threading.Tasks.Task t = System.Threading.Tasks.Task.Factory.StartNew(() => #endif { AppDomain jobDomain = null; if (TASKS_IN_APPDOMAINS) { System.AppDomainSetup appDomainSetup = new AppDomainSetup(); appDomainSetup.ShadowCopyFiles = "true"; appDomainSetup.ShadowCopyDirectories = _tasksFolder; appDomainSetup.CachePath = _tasksFolder; jobDomain = AppDomain.CreateDomain(Guid.NewGuid().ToString(), new System.Security.Policy.Evidence(AppDomain.CurrentDomain.Evidence), appDomainSetup); } try { object prevTaskOP = null; foreach (Technisient.SamayConfig.Task task in job.TaskChain.Task) { if (_TaskAssemblyDict.ContainsKey(task.ClassName)) { try { TaskBase TASK; if (TASKS_IN_APPDOMAINS) { TASK = CreateInstanceInAppDomain(task.ClassName, jobDomain); } else { TASK = (TaskBase)CreateInstance(task.ClassName); } if (task.param != null) // AssignParameters(TASK, task.param); TASK.AssignParameters(GetParameters(TASK, task.param)); tc.logLevel = task.LogLevel; tc.taskId = task.Id; TASK._taskContext = tc; TASK.InitListener("net.pipe://localhost/SamayEngineListener"); SamayLogger.LogInfo("Running " + task.ClassName, job.JobName, task.ClassName, task.Id); //FUTURE: Impersonation in config if (task.RetryOnError == null) { //try only once prevTaskOP = TASK.Run(prevTaskOP); } else { bool ranSucessfully = false; for (int i = 0; i < task.RetryOnError.RetryTimes; i++) { #region RetryOnError try { prevTaskOP = TASK.Run(prevTaskOP); ranSucessfully = true; break; } catch (Exception ex) { SamayLogger.LogWarning("Failed Executing " + task.ClassName + " in Job " + job.JobName + " (Attempt " + (i + 1).ToString() + " of " + task.RetryOnError.RetryTimes.ToString() + ")\n" + ex.ToString(), job.JobName, task.ClassName, task.Id); } Thread.Sleep(task.RetryOnError.RetryDelay_msec); #endregion RetryOnError } if (!ranSucessfully) throw new Exception("Aborting Execution of " + task.ClassName + " in Job " + job.JobName + " (Attempts Failed = " + task.RetryOnError.RetryTimes.ToString() + ")"); } } catch (Exception ex) { SamayLogger.LogError("Aborting execution of the Task. " + ex.ToString(), job.JobName, task.ClassName, task.Id); if (!job.TaskChain.ContinueOnError) break; } } else { SamayLogger.LogError("Unable to find assembly for " + task.ClassName + ". Task cannot be executed!", job.JobName, task.ClassName, task.Id); if (!job.TaskChain.ContinueOnError) break; } } #if ALLOW_CONCURRENT if (!job.AllowConcurrent) lock (_currentlyRunningJobs) { _currentlyRunningJobs[job.JobName].jobChainBusy = false; } #endif } catch (Exception ex) { SamayLogger.LogError(ex.ToString(), job.JobName, "Engine", job.Id); } finally { if (TASKS_IN_APPDOMAINS) { AppDomain.Unload(jobDomain); } } } #if USETASKS , TaskCreationOptions.LongRunning// | TaskCreationOptions.PreferFairness -- LOOKS GOOD WITHOUT USING FAIRNESS -- DONT USE /* Long-Running Tasks: * You may want to explicitly prevent a task from being put on a local queue. For example, you may know that a * particular work item will run for a relatively long time and is likely to block all other work items on * the local queue. In this case, you can specify the LongRunning option, which provides a hint to the scheduler * that an additional thread might be required for the task so that it does not block the forward progress of * other threads or work items on the local queue. By using this option you avoid the ThreadPool completely, * including the global and local queues. * */ ); return t; #else return null; #endif }
internal static void LogWarning(string logMsg) { SamayLogger.Log(logMsg, SamayLogger.SamayEngineLogJobName, "Engine", SamayLogger.SamayEngineLoggingGUID, LogLevel.Warn, DateTime.Now); }
public void SafeStopEngine(string requester) { SamayLogger.LogInfo("Engine Stop initiated by " + requester, "Engine Stop", null, "Engine Stop"); SamayEngine.SafeStopEngine(); }
public void ReloadConfig(string requester) { SamayLogger.LogInfo("Engine Config Reload initiated by " + requester, "Engine Stop", null, "Engine Stop"); SamayEngine.ReloadConfig(); }
public void Log(string logMsg, string job, string task, string id, LogLevel logLevel) { SamayLogger.Log(logMsg, job, task, id, logLevel, DateTime.Now); }