private async Task <LockInfo> WaitForLockImpl(Guid resource, TimeSpan?lockTimeout, CancellationToken cancellationToken)
{
LockInfo acquiredLock = LockInfo.Empty(resource);
while (!cancellationToken.IsCancellationRequested)
{
acquiredLock = await TryLockAsync(resource, lockTimeout);
if (acquiredLock.AsImmutable().HasLock())
{
break;
}
try
{
await Task.Delay(1000, cancellationToken);
}
catch (OperationCanceledException ex)
{
break;
}
}
return(acquiredLock);
}
private IList <TaskExecutionInfo> ProcessPendingTasks(int qty)
{
LockInfo lockInfo = null;
IList <TaskExecutionInfo> list = _EmtpyPendingTasks;
try
{
//Inter-Process lock
if (_taskManager.LockManager.TryLock(READ_PENDING_TASKS_LOCK, OneMinute, out lockInfo))
{
List <SingleTaskDefinition> pendingTasks;
using (var tr = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled))
using (var context = _taskManager.CreateRepository())
{
var utcNow = _taskManager.GetUtcNow().SetFractionalSecondPrecision(4);
var nowLimit = utcNow.AddMilliseconds(1);
pendingTasks = context.SingleTasks.AsNoTracking().Where(t =>
(t.Status == Distributed.SingleTaskStatus.Scheduled && t.ScheduledAt < nowLimit))
.OrderBy(t => t.ScheduledAt).ThenBy(t => t.Id)
.Take(qty).ToList();
}
list = pendingTasks.Select(t => ProcessTaskDefinition(t)).ToList();
}
}
finally
{
try { _taskManager.LockManager.Release(lockInfo); }
catch (Exception ex) { ex.LogException(); }
}
return(list);
}
private async Task <bool> releaseAsync(IDbConnectionAsync conn, LockInfo lockInfo)
{
var state = lockInfo.AsImmutable();
if (!state.HasLock())
{
return(true); //released already
}
var newLockRef = Math.Max(1, state.LockReference + 1);
using (var command = await DbAccessHelperAsync.CreateDbCommand(conn, RELEASE_LOCK))
{
command.Parameters.Add(new SqlParameter("newLockRef", newLockRef));
command.Parameters.Add(new SqlParameter("resource", state.Resource));
command.Parameters.Add(new SqlParameter("lockReference", state.LockReference));
command.Parameters.Add(new SqlParameter("lockedByPID", _PID));
try
{
await command.ExecuteNonQueryAsync();
lockInfo.Update(null, TimeSpan.Zero, 0, FrozenTimeClock.Instance);
return(true);
}
catch (Exception ex)
{
return(false);
}
}
}
private async Task <LockInfo> acquireLockAsync(IDbConnectionAsync conn, Guid resource, int lockReference, TimeSpan timeout)
{
var lockInfo = LockInfo.Empty(resource);
var newLockRef = Math.Max(1, lockReference + 1);
//only acquires lock if old lock reference number matches, and not locked yet.
using (var command = await DbAccessHelperAsync.CreateDbCommand(conn, ACQUIRE_lOCK_COMMANDS))
{
command.Parameters.Add(new SqlParameter("newLockRef", newLockRef));
command.Parameters.Add(new SqlParameter("timeout", Convert.ToInt64(timeout.TotalMilliseconds)));
command.Parameters.Add(new SqlParameter("lockedByMachine", _machineName));
command.Parameters.Add(new SqlParameter("lockedByPID", _PID));
command.Parameters.Add(new SqlParameter("resource", resource));
command.Parameters.Add(new SqlParameter("lockReference", lockReference));
try
{
var dbLockedTill = (DateTime?)await command.ExecuteScalarAsync();
if (dbLockedTill != null)
{
var clock = _clockProvider.GetClock();
lockInfo.Update(dbLockedTill, timeout, newLockRef, clock);
}
}
catch (Exception ex)
{
ex.LogException();
}
return(lockInfo);
}
}
public TaskExecutionInfo ProcessTaskDefinition(SingleTaskDefinition taskDefinition)
{
var lockInfo = LockInfo.Empty(taskDefinition.Identifier);
var taskTypeInfo = _taskManager.GetTaskInfo(taskDefinition.Identifier);
//links this task's cancellation token to the manager's token
var ctSource = CancellationTokenSource.CreateLinkedTokenSource(_listenerCtSource.Token);
try
{
//Inter-process lock. Ignores the task if it doesn't acquire lock
if (taskTypeInfo == null || !_taskManager.LockManager.TryRenewLock(lockInfo, taskTypeInfo.LockCycle, retryLock: true))
{
return(new TaskExecutionInfo(_managedThreadPool.CreateCompletedTask(), ctSource, lockInfo, taskDefinition));
}
var executionInfoLocal = new TaskExecutionInfo(null, ctSource, lockInfo, taskDefinition);
var taskBody = CreateTaskBody(executionInfoLocal, taskTypeInfo);
var task = _managedThreadPool.QueueUserWorkItem((o) => taskBody(), null);
executionInfoLocal.SetTask(task);
return(executionInfoLocal);
}
catch (Exception ex)
{
ex.LogException();
return(new TaskExecutionInfo(_managedThreadPool.CreateCompletedTask(), ctSource, lockInfo, taskDefinition));
}
}
public bool Release(LockInfo lockInfo)
{
EnsureNotDisposed();
if (lockInfo == null)
{
return(false);
}
return(TaskHelper.GetResultSafeSync(() => this.ReleaseAsync(lockInfo)));
}
private void CleanupAndRescheduleTasks()
{
List <RecurrentTaskProjection> recurrentTasks;
LockInfo lockInfo = null;
CleanupTasks();
try
{
//Inter-Process lock
if (_taskManager.LockManager.TryLock(RECURRENT_SINGLE_TASK_LOCK, OneMinute, out lockInfo))
{
using (var tr = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled))
using (var context = _taskManager.CreateRepository())
{
recurrentTasks = context.RecurrentTasks.AsNoTracking()
.Where(r => r.Enabled)
.Select(r => new RecurrentTaskProjection()
{
RecurrentTask = r,
LatestSingleTask = r.SingleTasks.OrderByDescending(t => t.ScheduledAt).Take(1).FirstOrDefault()
}).Where(a => a.LatestSingleTask == null ||
(a.LatestSingleTask.Status != Distributed.SingleTaskStatus.Scheduled && a.LatestSingleTask.Status != Distributed.SingleTaskStatus.Running))
.ToList();
}
//precision up to 4 decimals of a second
var utcNow = _taskManager.GetUtcNow().SetFractionalSecondPrecision(4);
foreach (var projection in recurrentTasks)
{
var taskInfo = _taskManager.GetTaskInfo(projection.RecurrentTask.Identifier);
if (taskInfo == null)
{
continue;
}
var interval = projection.RecurrentTask.Interval;
if (projection.LatestSingleTask == null || projection.LatestSingleTask.StatusChangedAt.AddMilliseconds(interval) < utcNow)
{
_taskManager.EnqueueSingleTask(taskInfo.Identifier, recurrentTaskId: projection.RecurrentTask.Id, delay: 0);
}
}
}
}
finally
{
try { _taskManager.LockManager.Release(lockInfo); }
catch (Exception ex) { ex.LogException(); }
}
}
public async Task <bool> ReleaseAsync(LockInfo lockInfo)
{
EnsureNotDisposed();
if (lockInfo == null)
{
return(false);
}
using (var tr = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled))
using (var conn = CreateConnection())
{
return(await releaseAsync(conn, lockInfo));
}
}
private async Task AwaitLockAndAwaitTaskEnd(Guid resource, TimeSpan lockTimeout, TimeSpan waitTime, TaskCompletionSource <ILockContextAsync> acquiredLockEvent, TaskCompletionSource <object> taskEndEvent)
{
try
{
using (var cancellationTS = new CancellationTokenSource(waitTime))
{
await _serialExecutionManager.RunAsync(resource, async() =>
{
LockInfo lockInfo = null;
try
{
lockInfo = await this.WaitForLockImpl(resource, lockTimeout, cancellationTS.Token);
if (lockInfo.AsImmutable().HasLock())
{
var lockContext = new LockContext(this, lockInfo);
acquiredLockEvent.SetResult(lockContext);
await taskEndEvent.Task;
}
else
{
throw new OperationCanceledException();
}
}
finally
{
this.Release(lockInfo);
}
}, cancellationTS.Token);
}
}
catch (Exception ex)
{
if (acquiredLockEvent.Task.Status != System.Threading.Tasks.TaskStatus.RanToCompletion)
{
if (ex is OperationCanceledException)
{
acquiredLockEvent.SetException(new TimeoutException($"Couldn't get lock on resource [{resource}]. Operation timed out."));
}
else
{
acquiredLockEvent.SetException(ex);
}
}
else
{
ex.LogException();
}
}
}
public async Task <LockInfo> TryLockAsync(Guid resource, TimeSpan?lockTimeout)
{
EnsureNotDisposed();
lockTimeout = lockTimeout ?? defaultTimeout;
verifyTimeoutLimits(lockTimeout.Value);
using (var tr = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled))
using (var conn = CreateConnection())
{
var lockReference = await GetFreeReferenceOrNewAsync(conn, resource);
//failed to insert record or resource already locked
if (lockReference == null)
{
return(LockInfo.Empty(resource));
}
//tries to acquire lock
return(await acquireLockAsync(conn, resource, lockReference.Value, lockTimeout.Value));
}
}
internal TaskExecutionInfo(IThreadTaskInfo task, CancellationTokenSource cancelTokenSource, LockInfo lockInfo, SingleTaskDefinition taskDefinition)
{
if (lockInfo == null)
{
throw new ArgumentNullException("lockInfo is null.");
}
if (taskDefinition == null)
{
throw new ArgumentNullException("taskDefinition is null.");
}
if (cancelTokenSource == null)
{
throw new ArgumentNullException("cancelTokenSource is null.");
}
//Task may be null when first initialized.
_task = task;
_lockInfo = lockInfo;
_taskDefinition = taskDefinition;
_cancelTokenSource = cancelTokenSource;
_lastAliveSignalAt = null;
}
public async Task <bool> TryRenewLockAsync(LockInfo lockInfo, TimeSpan?renewTimeout = null, bool retryLock = false)
{
EnsureNotDisposed();
if (lockInfo == null)
{
throw new ArgumentNullException("LockInfo is null");
}
//Allows only one task to run TryRenewLock on this lockInfo object
using (await lockInfo.Mutex.LockAsync())
{
renewTimeout = renewTimeout ?? lockInfo.AsImmutable().Timeout;
verifyTimeoutLimits(renewTimeout.Value);
using (var tr = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled))
using (var conn = CreateConnection())
{
if (await renewLockAsync(conn, renewTimeout.Value, lockInfo))
{
return(true);
}
}
}
if (retryLock)
{
LockInfo newLock = await this.TryLockAsync(lockInfo.Resource, renewTimeout.Value);
var newState = newLock.AsImmutable();
if (newState.HasLock())
{
var clock = _clockProvider.GetClock();
lockInfo.Update(newState.LockedUntil, newState.Timeout, newState.LockReference, clock);
return(true);
}
}
return(false);
}
private async Task <bool> renewLockAsync(IDbConnectionAsync conn, TimeSpan renewTimeout, LockInfo lockInfo)
{
var state = lockInfo.AsImmutable();
if (!state.HasLock())
{
return(false);
}
var newLockRef = Math.Max(1, state.LockReference + 1);
using (var command = await DbAccessHelperAsync.CreateDbCommand(conn, RENEW_LOCK_COMMANDS))
{
command.Parameters.Add(new SqlParameter("newLockRef", newLockRef));
command.Parameters.Add(new SqlParameter("resource", state.Resource));
command.Parameters.Add(new SqlParameter("lockReference", state.LockReference));
command.Parameters.Add(new SqlParameter("timeout", Convert.ToInt64(renewTimeout.TotalMilliseconds)));
command.Parameters.Add(new SqlParameter("lockedByPID", _PID));
try
{
var dbLockedTill = (DateTime?)await command.ExecuteScalarAsync();
if (dbLockedTill != null)
{
var clock = _clockProvider.GetClock();
lockInfo.Update(dbLockedTill, renewTimeout, newLockRef, clock);
return(lockInfo.AsImmutable().HasLock());
}
return(false);
}
catch (Exception ex)
{
return(false);
}
}
}
public bool TryRenewLock(LockInfo lockInfo, TimeSpan?renewTimeout = null, bool retryLock = false)
{
EnsureNotDisposed();
return(TaskHelper.GetResultSafeSync(() => TryRenewLockAsync(lockInfo, renewTimeout, retryLock)));
}
public LockContext(ILockManager lockmanager, LockInfo lockInfo)
{
this._lockmanager = lockmanager;
this._lockInfo = lockInfo;
}
public int EnqueueRecurrentTask(Guid identifier, long interval)
{
if (interval < 1000)
{
throw new ArgumentException("interval");
}
var info = this.GetTaskInfo(identifier);
if (info == null)
{
throw new ArgumentException(String.Format("Task identifier [{0}] not registered.", identifier.ToString()));
}
LockInfo lockInfo = null;
try
{
if (LockManager.WaitForLock(ENQUEUE_RECURRENT_TASK_LOCK, new TimeSpan(TimeSpan.TicksPerMinute), new TimeSpan(TimeSpan.TicksPerSecond * 10), out lockInfo))
{
using (var tr = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled))
using (var context = CreateRepository())
{
//Adds or updates a recurrent task
var data = context.RecurrentTasks
.Where(t => t.Identifier == info.Identifier)
.OrderBy(t => t.Id).FirstOrDefault();
data = data ?? new RecurrentTaskDefinition()
{
Identifier = info.Identifier
};
data.Enabled = true;
data.Interval = interval;
if (data.Id == 0)
{
context.RecurrentTasks.Add(data);
}
context.SaveChanges();
var copies = context.RecurrentTasks
.Where(t => t.Identifier == info.Identifier)
.OrderBy(t => t.Id).ToList();
if (copies.Count == 0)
{
throw new ApplicationException("Database task entry was modified while enqueuing task.");
}
//Recurrent tasks cannot have duplicates
foreach (var duplicate in copies.Skip(1))
{
context.RecurrentTasks.Remove(duplicate);
}
var first = copies.FirstOrDefault();
if (first.Interval != interval)
{
first.Interval = interval;
}
context.SaveChanges();
return(first.Id);
}
}
else
{
throw new ApplicationException("Lock for resource ENQUEUE_RECURRENT_TASK_LOCK has been constantly denied.");
}
}
finally
{
LockManager.Release(lockInfo);
}
}
public bool WaitForLock(Guid resource, TimeSpan?lockTimeout, TimeSpan waitTime, out LockInfo lockInfo)
{
EnsureNotDisposed();
lockInfo = TaskHelper.GetResultSafeSync(() => this.WaitForLockAsync(resource, lockTimeout, waitTime));
return(lockInfo.AsImmutable().HasLock());
}
public bool TryLock(Guid resource, TimeSpan?lockTimeout, out LockInfo lockInfo)
{
EnsureNotDisposed();
lockInfo = TaskHelper.GetResultSafeSync(() => TryLockAsync(resource, lockTimeout));
return(lockInfo.AsImmutable().HasLock());
}
