public ClientConnectionSettings GetConnection(TaskId taskId)
{
lock (sync)
{
if (_connections.ContainsKey(taskId))
return _connections[taskId];
return null;
}
}
public void SetConnection(TaskId taskId, ClientConnectionSettings connectionSettings)
{
lock (sync)
{
if (_connections.ContainsKey(taskId))
_connections[taskId] = connectionSettings;
else
_connections.Add(taskId, connectionSettings);
}
}
public void SetLastCleaned(TaskId taskId)
{
using (var connection = CreateNewConnection(taskId))
{
using (var command = new SqlCommand(TaskQueryBuilder.SetLastCleanUpTimeQuery, connection))
{
command.Parameters.Add(new SqlParameter("@ApplicationName", SqlDbType.VarChar, 200)).Value = taskId.ApplicationName;
command.Parameters.Add(new SqlParameter("@TaskName", SqlDbType.VarChar, 200)).Value = taskId.TaskName;
command.ExecuteNonQuery();
}
}
}
public IList<ProtoListBlockItem> GetListBlockItems(TaskId taskId, string listBlockId)
{
var results = new List<ProtoListBlockItem>();
try
{
using (var connection = CreateNewConnection(taskId))
{
var command = connection.CreateCommand();
command.CommandText = ListBlockQueryBuilder.GetListBlockItems;
command.CommandTimeout = ConnectionStore.Instance.GetConnection(taskId).QueryTimeoutSeconds;
command.Parameters.Add("@BlockId", SqlDbType.BigInt).Value = long.Parse(listBlockId);
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
var listBlock = new ProtoListBlockItem();
listBlock.ListBlockItemId = reader["ListBlockItemId"].ToString();
listBlock.Value = SerializedValueReader.ReadValueAsString(reader, "Value", "CompressedValue");
listBlock.Status = (ItemStatus)int.Parse(reader["Status"].ToString());
if (reader["LastUpdated"] == DBNull.Value)
listBlock.LastUpdated = DateTime.MinValue;
else
listBlock.LastUpdated = reader.GetDateTime(5);
if (reader["StatusReason"] == DBNull.Value)
listBlock.StatusReason = null;
else
listBlock.StatusReason = reader.GetString(6);
if (reader["Step"] == DBNull.Value)
listBlock.Step = null;
else
listBlock.Step = reader.GetByte(7);
results.Add(listBlock);
}
}
}
}
catch (SqlException sqlEx)
{
if (TransientErrorDetector.IsTransient(sqlEx))
throw new TransientException("A transient exception has occurred", sqlEx);
throw;
}
return results;
}
protected SqlConnection CreateNewConnection(TaskId taskId)
{
try
{
var connection = new SqlConnection(ConnectionStore.Instance.GetConnection(taskId).ConnectionString);
connection.Open();
return connection;
}
catch (SqlException sqlEx)
{
if (TransientErrorDetector.IsTransient(sqlEx))
throw new TransientException("A transient exception has occurred", sqlEx);
throw;
}
}
public DateTime GetLastTaskCleanUpTime(TaskId taskId)
{
using (var connection = CreateNewConnection(taskId))
{
using (var command = new SqlCommand(TaskQueryBuilder.GetLastCleanUpTimeQuery, connection))
{
command.Parameters.Add(new SqlParameter("@ApplicationName", SqlDbType.VarChar, 200)).Value = taskId.ApplicationName;
command.Parameters.Add(new SqlParameter("@TaskName", SqlDbType.VarChar, 200)).Value = taskId.TaskName;
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
if (reader["LastCleaned"] == DBNull.Value)
return DateTime.MinValue;
return DateTime.Parse(reader["LastCleaned"].ToString());
}
}
}
}
return DateTime.MinValue;
}
private CompleteCriticalSectionResponse ReturnCriticalSectionToken(TaskId taskId, int taskDefinitionId, string taskExecutionId, CriticalSectionType criticalSectionType)
{
var response = new CompleteCriticalSectionResponse();
using (var connection = CreateNewConnection(taskId))
{
SqlTransaction transaction = connection.BeginTransaction(IsolationLevel.Serializable);
var command = connection.CreateCommand();
command.Transaction = transaction;
if (criticalSectionType == CriticalSectionType.User)
command.CommandText = TokensQueryBuilder.ReturnUserCriticalSectionTokenQuery;
else
command.CommandText = TokensQueryBuilder.ReturnClientCriticalSectionTokenQuery;
command.CommandTimeout = ConnectionStore.Instance.GetConnection(taskId).QueryTimeoutSeconds;
command.Parameters.Add("@TaskDefinitionId", SqlDbType.Int).Value = taskDefinitionId;
command.Parameters.Add("@TaskExecutionId", SqlDbType.Int).Value = int.Parse(taskExecutionId);
try
{
command.ExecuteNonQuery();
transaction.Commit();
}
catch (SqlException sqlEx)
{
TryRollBack(transaction, sqlEx);
}
catch (Exception ex)
{
TryRollback(transaction, ex);
}
}
return response;
}
public TaskDefinition EnsureTaskDefinition(TaskId taskId)
{
lock (_getTaskObj)
{
var taskDefinition = GetTask(taskId);
if (taskDefinition != null)
{
return taskDefinition;
}
else
{
// wait a random amount of time in case two threads or two instances of this repository
// independently belive that the task doesn't exist
Thread.Sleep(new Random(Guid.NewGuid().GetHashCode()).Next(2000));
taskDefinition = GetTask(taskId);
if (taskDefinition != null)
{
return taskDefinition;
}
return InsertNewTask(taskId);
}
}
}
private TaskDefinition InsertNewTask(TaskId taskId)
{
using (var connection = CreateNewConnection(taskId))
{
using (var command = new SqlCommand(TaskQueryBuilder.InsertTaskQuery, connection))
{
command.Parameters.Add(new SqlParameter("@ApplicationName", SqlDbType.VarChar, 200)).Value = taskId.ApplicationName;
command.Parameters.Add(new SqlParameter("@TaskName", SqlDbType.VarChar, 200)).Value = taskId.TaskName;
var task = new TaskDefinition();
task.TaskDefinitionId = (int)command.ExecuteScalar();
string key = taskId.ApplicationName + "::" + taskId.TaskName;
lock (_myCacheSyncObj)
{
CacheTaskDefinition(key, task);
}
return task;
}
}
}
private TaskDefinition LoadTask(TaskId taskId)
{
using (var connection = CreateNewConnection(taskId))
{
using (var command = new SqlCommand(TaskQueryBuilder.GetTaskQuery, connection))
{
command.Parameters.Add(new SqlParameter("@ApplicationName", SqlDbType.VarChar, 200)).Value = taskId.ApplicationName;
command.Parameters.Add(new SqlParameter("@TaskName", SqlDbType.VarChar, 200)).Value = taskId.TaskName;
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
var task = new TaskDefinition();
task.TaskDefinitionId = int.Parse(reader["TaskDefinitionId"].ToString());
return task;
}
}
}
}
return null;
}
private void RegisterEvent(TaskId taskId, string taskExecutionId, EventType eventType, string message)
{
_eventsRepository.LogEvent(taskId, taskExecutionId, eventType, message);
}
private void SetTaskExecutionAsFailed(TaskId taskId, string taskExecutionId)
{
using (var connection = CreateNewConnection(taskId))
{
using (var command = new SqlCommand(TaskQueryBuilder.SetTaskExecutionAsFailedQuery, connection))
{
command.CommandTimeout = ConnectionStore.Instance.GetConnection(taskId).QueryTimeoutSeconds;
command.Parameters.Add(new SqlParameter("@TaskExecutionId", SqlDbType.Int)).Value = int.Parse(taskExecutionId);
command.ExecuteNonQuery();
}
}
}
private int CreateOverrideTaskExecution(TaskId taskId, int taskDefinitionId, TimeSpan overrideThreshold, string referenceValue,
short failedTaskRetryLimit, short deadTaskRetryLimit, string tasklingVersion, string executionHeader)
{
using (var connection = CreateNewConnection(taskId))
{
using (var command = new SqlCommand(TaskQueryBuilder.InsertKeepAliveTaskExecution, connection))
{
command.CommandText = TaskQueryBuilder.InsertOverrideTaskExecution;
command.Parameters.Clear();
command.Parameters.Add(new SqlParameter("@TaskDefinitionId", SqlDbType.Int)).Value = taskDefinitionId;
command.Parameters.Add(new SqlParameter("@ServerName", SqlDbType.VarChar, 200)).Value = Environment.MachineName;
command.Parameters.Add(new SqlParameter("@TaskDeathMode", SqlDbType.TinyInt)).Value = (byte)TaskDeathMode.Override;
command.Parameters.Add(new SqlParameter("@OverrideThreshold", SqlDbType.Time)).Value = overrideThreshold;
command.Parameters.Add(new SqlParameter("@FailedTaskRetryLimit", SqlDbType.SmallInt)).Value = failedTaskRetryLimit;
command.Parameters.Add(new SqlParameter("@DeadTaskRetryLimit", SqlDbType.SmallInt)).Value = deadTaskRetryLimit;
command.Parameters.Add(new SqlParameter("@TasklingVersion", SqlDbType.VarChar)).Value = tasklingVersion;
if (executionHeader == null)
command.Parameters.Add(new SqlParameter("@ExecutionHeader", SqlDbType.NVarChar, -1)).Value = DBNull.Value;
else
command.Parameters.Add(new SqlParameter("@ExecutionHeader", SqlDbType.NVarChar, -1)).Value = executionHeader;
if (referenceValue == null)
command.Parameters.Add(new SqlParameter("@ReferenceValue", SqlDbType.NVarChar, 2000)).Value = DBNull.Value;
else
command.Parameters.Add(new SqlParameter("@ReferenceValue", SqlDbType.NVarChar, 2000)).Value = referenceValue;
return (int)command.ExecuteScalar();
}
}
}
private int CreateKeepAliveTaskExecution(TaskId taskId, int taskDefinitionId, TimeSpan keepAliveInterval, TimeSpan keepAliveDeathThreshold, string referenceValue,
short failedTaskRetryLimit, short deadTaskRetryLimit, string tasklingVersion, string executionHeader)
{
using (var connection = CreateNewConnection(taskId))
{
using (var command = new SqlCommand(TaskQueryBuilder.InsertKeepAliveTaskExecution, connection))
{
command.CommandTimeout = ConnectionStore.Instance.GetConnection(taskId).QueryTimeoutSeconds;
command.Parameters.Add(new SqlParameter("@TaskDefinitionId", SqlDbType.Int)).Value = taskDefinitionId;
command.Parameters.Add(new SqlParameter("@ServerName", SqlDbType.VarChar, 200)).Value = Environment.MachineName;
command.Parameters.Add(new SqlParameter("@TaskDeathMode", SqlDbType.TinyInt)).Value = (byte)TaskDeathMode.KeepAlive;
command.Parameters.Add(new SqlParameter("@KeepAliveInterval", SqlDbType.Time)).Value = keepAliveInterval;
command.Parameters.Add(new SqlParameter("@KeepAliveDeathThreshold", SqlDbType.Time)).Value = keepAliveDeathThreshold;
command.Parameters.Add(new SqlParameter("@FailedTaskRetryLimit", SqlDbType.SmallInt)).Value = failedTaskRetryLimit;
command.Parameters.Add(new SqlParameter("@DeadTaskRetryLimit", SqlDbType.SmallInt)).Value = deadTaskRetryLimit;
command.Parameters.Add(new SqlParameter("@TasklingVersion", SqlDbType.VarChar)).Value = tasklingVersion;
if (executionHeader == null)
command.Parameters.Add(new SqlParameter("@ExecutionHeader", SqlDbType.NVarChar, -1)).Value = DBNull.Value;
else
command.Parameters.Add(new SqlParameter("@ExecutionHeader", SqlDbType.NVarChar, -1)).Value = executionHeader;
if (referenceValue == null)
command.Parameters.Add(new SqlParameter("@ReferenceValue", SqlDbType.NVarChar, 2000)).Value = DBNull.Value;
else
command.Parameters.Add(new SqlParameter("@ReferenceValue", SqlDbType.NVarChar, 2000)).Value = referenceValue;
var taskExecutionId = (int)command.ExecuteScalar();
return taskExecutionId;
}
}
}
private TaskExecutionStartResponse TryGetExecutionToken(TaskId taskId, int taskDefinitionId, int taskExecutionId, int concurrencyLimit)
{
var tokenRequest = new TokenRequest()
{
TaskId = taskId,
TaskDefinitionId = taskDefinitionId,
TaskExecutionId = taskExecutionId.ToString(),
ConcurrencyLimit = concurrencyLimit
};
var tokenResponse = _executionTokenRepository.TryAcquireExecutionToken(tokenRequest);
var response = new TaskExecutionStartResponse();
response.ExecutionTokenId = tokenResponse.ExecutionTokenId;
response.GrantStatus = tokenResponse.GrantStatus;
response.StartedAt = tokenResponse.StartedAt;
response.TaskExecutionId = taskExecutionId.ToString();
return response;
}
private TaskDefinition GetTask(TaskId taskId)
{
return GetCachedDefinition(taskId);
}
private TaskDefinition GetCachedDefinition(TaskId taskId)
{
lock (_myCacheSyncObj)
{
string key = taskId.ApplicationName + "::" + taskId.TaskName;
if (_cachedTaskDefinitions.ContainsKey(key))
{
var definition = _cachedTaskDefinitions[key];
if ((definition.CachedAt - DateTime.UtcNow).TotalSeconds < 300)
return definition.TaskDefinition;
}
else
{
var task = LoadTask(taskId);
CacheTaskDefinition(key, task);
return task;
}
}
return null;
}
private bool TryAcquireCriticalSection(TaskId taskId, int taskDefinitionId, string taskExecutionId, CriticalSectionType criticalSectionType)
{
bool granted = false;
using (var connection = CreateNewConnection(taskId))
{
SqlTransaction transaction = connection.BeginTransaction(IsolationLevel.Serializable);
var command = connection.CreateCommand();
command.Transaction = transaction;
command.CommandTimeout = ConnectionStore.Instance.GetConnection(taskId).QueryTimeoutSeconds; ;
try
{
AcquireRowLock(taskDefinitionId, taskExecutionId, command);
var csState = GetCriticalSectionState(taskDefinitionId, criticalSectionType, command);
CleanseOfExpiredExecutions(csState, command);
if (csState.IsGranted)
{
// if the critical section is still granted to another execution after cleansing
// then we rejected the request. If the execution is not in the queue then we add it
if (!csState.ExistsInQueue(taskExecutionId))
csState.AddToQueue(taskExecutionId);
granted = false;
}
else
{
if (csState.GetQueue().Any())
{
if (csState.GetFirstExecutionIdInQueue() == taskExecutionId)
{
GrantCriticalSection(csState, taskDefinitionId, taskExecutionId, command);
csState.RemoveFirstInQueue();
granted = true;
}
else
{
// not next in queue so cannot be granted the critical section
granted = false;
}
}
else
{
GrantCriticalSection(csState, taskDefinitionId, taskExecutionId, command);
granted = true;
}
}
if (csState.HasBeenModified)
UpdateCriticalSectionState(taskDefinitionId, csState, criticalSectionType, command);
transaction.Commit();
}
catch (SqlException sqlEx)
{
TryRollBack(transaction, sqlEx);
}
catch (Exception ex)
{
TryRollback(transaction, ex);
}
}
return granted;
}
