protected override string DequeueJson(Worker worker, int timeoutSeconds)
{
string data;
if (_Queue.TryTake(out data, timeoutSeconds * 1000))
{
return data;
}
return null;
}
public string GetInProgressJson(Worker worker)
{
var connection = GetOpenConnection();
string data = connection.Lists.GetString(0, GetRedisKey("worker:{0}:inprogress", GetWorkerKey(worker)), 0).Result;
if (data != null)
{
try
{
connection.Hashes.Set(0, GetRedisKey("worker:{0}:state", GetWorkerKey(worker)), "currentstart", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture));
}
catch (Exception ex)
{
if (Roque.Core.RoqueTrace.Switch.TraceError)
{
Trace.TraceError("[REDIS] error registering job start: " + ex.Message, ex);
}
}
}
return data;
}
public void RegisterSubscribersForWorker(Worker worker)
{
if (string.IsNullOrEmpty(worker.Name))
{
return;
}
var workerConfig = Configuration.Roque.Settings.Workers[worker.Name];
if (workerConfig == null || workerConfig.Subscribers.Count < 1)
{
return;
}
foreach (var subscriberConfig in workerConfig.Subscribers.OfType<SubscriberElement>())
{
try
{
string sourceQueue = subscriberConfig.SourceQueue;
if (string.IsNullOrEmpty(sourceQueue))
{
sourceQueue = Queue.DefaultEventQueueName;
}
RegisterSubscriber(Activator.CreateInstance(Type.GetType(subscriberConfig.SubscriberType)), sourceQueue, worker.Queue.Name);
}
catch (Exception ex)
{
if (RoqueTrace.Switch.TraceError)
{
Trace.TraceError(string.Format("Error registering subscriber: {0}. Type: {1}", ex.Message, subscriberConfig.SubscriberType), ex);
}
throw;
}
}
}
protected string GetWorkerKey(Worker worker)
{
return string.Format("w_{0}_{1}", worker.Name, worker.ID);
}
protected override string DequeueJson(Worker worker, int timeoutSeconds)
{
var connection = Connection.GetOpen();
// move job from queue to worker in progress
string data = connection.Lists.BlockingRemoveLastAndAddFirstString(0, GetRedisKey(), GetRedisKey("worker:{0}:inprogress", GetWorkerKey(worker)), timeoutSeconds).Result;
if (!string.IsNullOrEmpty(data))
{
try
{
connection.Hashes.Set(0, GetRedisKey("worker:{0}:state", GetWorkerKey(worker)), "currentstart", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture));
}
catch (Exception ex)
{
RoqueTrace.Source.Trace(TraceEventType.Error, "[REDIS] error registering job start: {0}", ex.Message, ex);
}
}
return data;
}
public void JobCompleted(Worker worker, Job job, bool failed)
{
try
{
var connection = Connection.GetOpen();
string json = connection.Lists.RemoveFirstString(0, GetRedisKey("worker:{0}:inprogress", GetWorkerKey(worker))).Result;
if (failed)
{
connection.Lists.AddFirst(0, GetRedisKey("failed"), json).Wait();
}
connection.Hashes.Remove(0, GetRedisKey("worker:{0}:state", GetWorkerKey(worker)), "currentstart").Wait();
connection.Hashes.Set(0, GetRedisKey("worker:{0}:state", GetWorkerKey(worker)), "lastcomplete", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture)).Wait();
connection.Hashes.Set(0, GetRedisKey("state"), "lastcomplete", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture)).Wait();
}
catch (Exception ex)
{
RoqueTrace.Source.Trace(TraceEventType.Error, "[REDIS] error registering job completion: {0}", ex.Message, ex);
throw;
}
}
/// <summary>
/// Get a worker by name
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public static Worker Get(string name)
{
Worker worker;
if (string.IsNullOrWhiteSpace(name))
{
name = string.Empty;
}
if (!_Instances.TryGetValue(name, out worker))
{
try
{
var workersConfig = Configuration.Roque.Settings.Workers;
if (workersConfig == null)
{
throw new Exception("No workers found in configuration");
}
var workerConfig = workersConfig[name];
if (workerConfig == null)
{
throw new Exception("Worker not found: " + name);
}
worker = new Worker(workerConfig.Name, Queue.Get(workerConfig.Queue), workerConfig.TooManyErrors, workerConfig.TooManyErrorsRetrySeconds);
worker.AutoStart = workerConfig.AutoStart;
}
catch (Exception ex)
{
if (RoqueTrace.Switch.TraceError)
{
Trace.TraceError(ex.Message, ex);
}
throw;
}
}
return worker;
}
protected abstract string DequeueJson(Worker worker, int timeoutSeconds);
/// <summary>
/// Get the job that is currently marked as in progress for a worker.
/// </summary>
/// <param name="worker"></param>
/// <returns>an in progress job, or null if none was found</returns>
public Job GetInProgressJob(Worker worker)
{
try
{
RoqueTrace.Source.Trace(TraceEventType.Verbose, "Looking for pending jobs on {0} queue", Name);
IQueueWithInProgressData queueWithInProgress = this as IQueueWithInProgressData;
if (queueWithInProgress == null)
{
// this queue doesn't support resuming work in progress
return null;
}
string data = queueWithInProgress.GetInProgressJson(worker);
if (string.IsNullOrEmpty(data))
{
return null;
}
Job job = JsonConvert.DeserializeObject<Job>(data);
job.MarkAsResuming();
RoqueTrace.Source.Trace(TraceEventType.Verbose, "Resuming pending job on {0} queue", Name);
return job;
}
catch (Exception ex)
{
RoqueTrace.Source.Trace(TraceEventType.Error, "Error receiving in progress job: {0}", ex.Message, ex);
throw;
}
}
/// <summary>
/// Picks a job from this queue. If there queue is empty blocks until one is obtained.
/// </summary>
/// <param name="worker">the worker that requests the job, the job will me marked as in progress for this worker</param>
/// <param name="timeoutSeconds">if blocked waiting for a job after this time null will be return</param>
/// <returns>a job or null if none could be obtained before timeout</returns>
public Job Dequeue(Worker worker, int timeoutSeconds = 10)
{
try
{
if (timeoutSeconds < 1)
{
timeoutSeconds = 1;
}
RoqueTrace.Source.Trace(TraceEventType.Verbose, "Worker {0} waiting job from {1} queue...", worker.Name, Name);
string data = DequeueJson(worker, timeoutSeconds);
if (string.IsNullOrEmpty(data))
{
return null;
}
Job job = JsonConvert.DeserializeObject<Job>(data);
RoqueTrace.Source.Trace(TraceEventType.Verbose, "Worker {0} received job from {1} queue", worker.Name, Name);
return job;
}
catch (Exception ex)
{
RoqueTrace.Source.Trace(TraceEventType.Error, "Error receiving job: {0}", ex.Message, ex);
throw;
}
}
/// <summary>
/// Mark a job as completed by a worker, removing it from in progress.
/// </summary>
/// <param name="worker"></param>
/// <param name="job"></param>
public void Completed(Worker worker, Job job, bool failed = false)
{
IQueueWithInProgressData queueWithInProgress = this as IQueueWithInProgressData;
if (queueWithInProgress != null)
{
queueWithInProgress.JobCompleted(worker, job, failed);
}
}
/// <summary>
/// Get the job that is currently marked as in progress for a worker.
/// </summary>
/// <param name="worker"></param>
/// <returns>an in progress job, or null if none was found</returns>
public Job GetInProgressJob(Worker worker)
{
try
{
if (RoqueTrace.Switch.TraceVerbose)
{
Trace.TraceInformation(string.Format("Looking for pending jobs on {0}", Name));
}
IQueueWithInProgressData queueWithInProgress = this as IQueueWithInProgressData;
if (queueWithInProgress == null)
{
// this queue doesn't support resuming work in progress
return null;
}
string data = queueWithInProgress.GetInProgressJson(worker);
if (string.IsNullOrEmpty(data))
{
return null;
}
Job job = JsonConvert.DeserializeObject<Job>(data);
job.MarkAsResuming();
if (RoqueTrace.Switch.TraceVerbose)
{
Trace.TraceInformation(string.Format("Resuming pending job on {0}", Name));
}
return job;
}
catch (Exception ex)
{
if (RoqueTrace.Switch.TraceError)
{
Trace.TraceError("Error dequeuing in progress job: " + ex.Message, ex);
}
throw;
}
}
/// <summary>
/// Picks a job from this queue. If there queue is empty blocks until one is obtained.
/// </summary>
/// <param name="worker">the worker that requests the job, the job will me marked as in progress for this worker</param>
/// <param name="timeoutSeconds">if blocked waiting for a job after this time null will be return</param>
/// <returns>a job or null if none could be obtained before timeout</returns>
public Job Dequeue(Worker worker, int timeoutSeconds = 10)
{
try
{
if (timeoutSeconds < 1)
{
timeoutSeconds = 1;
}
if (RoqueTrace.Switch.TraceVerbose)
{
Trace.TraceInformation(string.Format("Worker {0} waiting job on {0}", worker.Name, Name));
}
string data = DequeueJson(worker, timeoutSeconds);
if (string.IsNullOrEmpty(data))
{
return null;
}
Job job = JsonConvert.DeserializeObject<Job>(data);
if (RoqueTrace.Switch.TraceVerbose)
{
Trace.TraceInformation(string.Format("Worker {0} received job from {1}", worker.Name, Name));
}
return job;
}
catch (Exception ex)
{
if (RoqueTrace.Switch.TraceError)
{
Trace.TraceError("Error dequeuing job: " + ex.Message, ex);
}
throw;
}
}
