protected override string DequeueJson(Worker worker, int timeoutSeconds) { string data; if (_Queue.TryTake(out data, timeoutSeconds * 1000)) { return data; } return null; }
public string GetInProgressJson(Worker worker) { var connection = GetOpenConnection(); string data = connection.Lists.GetString(0, GetRedisKey("worker:{0}:inprogress", GetWorkerKey(worker)), 0).Result; if (data != null) { try { connection.Hashes.Set(0, GetRedisKey("worker:{0}:state", GetWorkerKey(worker)), "currentstart", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture)); } catch (Exception ex) { if (Roque.Core.RoqueTrace.Switch.TraceError) { Trace.TraceError("[REDIS] error registering job start: " + ex.Message, ex); } } } return data; }
public void RegisterSubscribersForWorker(Worker worker) { if (string.IsNullOrEmpty(worker.Name)) { return; } var workerConfig = Configuration.Roque.Settings.Workers[worker.Name]; if (workerConfig == null || workerConfig.Subscribers.Count < 1) { return; } foreach (var subscriberConfig in workerConfig.Subscribers.OfType<SubscriberElement>()) { try { string sourceQueue = subscriberConfig.SourceQueue; if (string.IsNullOrEmpty(sourceQueue)) { sourceQueue = Queue.DefaultEventQueueName; } RegisterSubscriber(Activator.CreateInstance(Type.GetType(subscriberConfig.SubscriberType)), sourceQueue, worker.Queue.Name); } catch (Exception ex) { if (RoqueTrace.Switch.TraceError) { Trace.TraceError(string.Format("Error registering subscriber: {0}. Type: {1}", ex.Message, subscriberConfig.SubscriberType), ex); } throw; } } }
protected string GetWorkerKey(Worker worker) { return string.Format("w_{0}_{1}", worker.Name, worker.ID); }
protected override string DequeueJson(Worker worker, int timeoutSeconds) { var connection = Connection.GetOpen(); // move job from queue to worker in progress string data = connection.Lists.BlockingRemoveLastAndAddFirstString(0, GetRedisKey(), GetRedisKey("worker:{0}:inprogress", GetWorkerKey(worker)), timeoutSeconds).Result; if (!string.IsNullOrEmpty(data)) { try { connection.Hashes.Set(0, GetRedisKey("worker:{0}:state", GetWorkerKey(worker)), "currentstart", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture)); } catch (Exception ex) { RoqueTrace.Source.Trace(TraceEventType.Error, "[REDIS] error registering job start: {0}", ex.Message, ex); } } return data; }
public void JobCompleted(Worker worker, Job job, bool failed) { try { var connection = Connection.GetOpen(); string json = connection.Lists.RemoveFirstString(0, GetRedisKey("worker:{0}:inprogress", GetWorkerKey(worker))).Result; if (failed) { connection.Lists.AddFirst(0, GetRedisKey("failed"), json).Wait(); } connection.Hashes.Remove(0, GetRedisKey("worker:{0}:state", GetWorkerKey(worker)), "currentstart").Wait(); connection.Hashes.Set(0, GetRedisKey("worker:{0}:state", GetWorkerKey(worker)), "lastcomplete", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture)).Wait(); connection.Hashes.Set(0, GetRedisKey("state"), "lastcomplete", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture)).Wait(); } catch (Exception ex) { RoqueTrace.Source.Trace(TraceEventType.Error, "[REDIS] error registering job completion: {0}", ex.Message, ex); throw; } }
/// <summary> /// Get a worker by name /// </summary> /// <param name="name"></param> /// <returns></returns> public static Worker Get(string name) { Worker worker; if (string.IsNullOrWhiteSpace(name)) { name = string.Empty; } if (!_Instances.TryGetValue(name, out worker)) { try { var workersConfig = Configuration.Roque.Settings.Workers; if (workersConfig == null) { throw new Exception("No workers found in configuration"); } var workerConfig = workersConfig[name]; if (workerConfig == null) { throw new Exception("Worker not found: " + name); } worker = new Worker(workerConfig.Name, Queue.Get(workerConfig.Queue), workerConfig.TooManyErrors, workerConfig.TooManyErrorsRetrySeconds); worker.AutoStart = workerConfig.AutoStart; } catch (Exception ex) { if (RoqueTrace.Switch.TraceError) { Trace.TraceError(ex.Message, ex); } throw; } } return worker; }
protected abstract string DequeueJson(Worker worker, int timeoutSeconds);
/// <summary> /// Get the job that is currently marked as in progress for a worker. /// </summary> /// <param name="worker"></param> /// <returns>an in progress job, or null if none was found</returns> public Job GetInProgressJob(Worker worker) { try { RoqueTrace.Source.Trace(TraceEventType.Verbose, "Looking for pending jobs on {0} queue", Name); IQueueWithInProgressData queueWithInProgress = this as IQueueWithInProgressData; if (queueWithInProgress == null) { // this queue doesn't support resuming work in progress return null; } string data = queueWithInProgress.GetInProgressJson(worker); if (string.IsNullOrEmpty(data)) { return null; } Job job = JsonConvert.DeserializeObject<Job>(data); job.MarkAsResuming(); RoqueTrace.Source.Trace(TraceEventType.Verbose, "Resuming pending job on {0} queue", Name); return job; } catch (Exception ex) { RoqueTrace.Source.Trace(TraceEventType.Error, "Error receiving in progress job: {0}", ex.Message, ex); throw; } }
/// <summary> /// Picks a job from this queue. If there queue is empty blocks until one is obtained. /// </summary> /// <param name="worker">the worker that requests the job, the job will me marked as in progress for this worker</param> /// <param name="timeoutSeconds">if blocked waiting for a job after this time null will be return</param> /// <returns>a job or null if none could be obtained before timeout</returns> public Job Dequeue(Worker worker, int timeoutSeconds = 10) { try { if (timeoutSeconds < 1) { timeoutSeconds = 1; } RoqueTrace.Source.Trace(TraceEventType.Verbose, "Worker {0} waiting job from {1} queue...", worker.Name, Name); string data = DequeueJson(worker, timeoutSeconds); if (string.IsNullOrEmpty(data)) { return null; } Job job = JsonConvert.DeserializeObject<Job>(data); RoqueTrace.Source.Trace(TraceEventType.Verbose, "Worker {0} received job from {1} queue", worker.Name, Name); return job; } catch (Exception ex) { RoqueTrace.Source.Trace(TraceEventType.Error, "Error receiving job: {0}", ex.Message, ex); throw; } }
/// <summary> /// Mark a job as completed by a worker, removing it from in progress. /// </summary> /// <param name="worker"></param> /// <param name="job"></param> public void Completed(Worker worker, Job job, bool failed = false) { IQueueWithInProgressData queueWithInProgress = this as IQueueWithInProgressData; if (queueWithInProgress != null) { queueWithInProgress.JobCompleted(worker, job, failed); } }
/// <summary> /// Get the job that is currently marked as in progress for a worker. /// </summary> /// <param name="worker"></param> /// <returns>an in progress job, or null if none was found</returns> public Job GetInProgressJob(Worker worker) { try { if (RoqueTrace.Switch.TraceVerbose) { Trace.TraceInformation(string.Format("Looking for pending jobs on {0}", Name)); } IQueueWithInProgressData queueWithInProgress = this as IQueueWithInProgressData; if (queueWithInProgress == null) { // this queue doesn't support resuming work in progress return null; } string data = queueWithInProgress.GetInProgressJson(worker); if (string.IsNullOrEmpty(data)) { return null; } Job job = JsonConvert.DeserializeObject<Job>(data); job.MarkAsResuming(); if (RoqueTrace.Switch.TraceVerbose) { Trace.TraceInformation(string.Format("Resuming pending job on {0}", Name)); } return job; } catch (Exception ex) { if (RoqueTrace.Switch.TraceError) { Trace.TraceError("Error dequeuing in progress job: " + ex.Message, ex); } throw; } }
/// <summary> /// Picks a job from this queue. If there queue is empty blocks until one is obtained. /// </summary> /// <param name="worker">the worker that requests the job, the job will me marked as in progress for this worker</param> /// <param name="timeoutSeconds">if blocked waiting for a job after this time null will be return</param> /// <returns>a job or null if none could be obtained before timeout</returns> public Job Dequeue(Worker worker, int timeoutSeconds = 10) { try { if (timeoutSeconds < 1) { timeoutSeconds = 1; } if (RoqueTrace.Switch.TraceVerbose) { Trace.TraceInformation(string.Format("Worker {0} waiting job on {0}", worker.Name, Name)); } string data = DequeueJson(worker, timeoutSeconds); if (string.IsNullOrEmpty(data)) { return null; } Job job = JsonConvert.DeserializeObject<Job>(data); if (RoqueTrace.Switch.TraceVerbose) { Trace.TraceInformation(string.Format("Worker {0} received job from {1}", worker.Name, Name)); } return job; } catch (Exception ex) { if (RoqueTrace.Switch.TraceError) { Trace.TraceError("Error dequeuing job: " + ex.Message, ex); } throw; } }