private void MergeSimilarTasks(DatabaseTask task, Etag taskId, Action <IComparable> updateMaxTaskId)
{
var type = task.GetType().FullName;
var tasksByIndexAndType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByIndexAndType);
using (var iterator = tasksByIndexAndType.MultiRead(Snapshot, (Slice)CreateKey(task.Index, type)))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
return;
}
var totalKeysToProcess = task.NumberOfKeys;
do
{
if (totalKeysToProcess >= 5 * 1024)
{
break;
}
var currentId = Etag.Parse(iterator.CurrentKey.ToString());
// this is the same task that we are trying to merge
if (currentId == taskId)
{
continue;
}
ushort version;
var value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null)
{
continue;
}
DatabaseTask existingTask;
try
{
existingTask = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
}
catch (Exception e)
{
Logger.ErrorException(string.Format("Could not create instance of a task: {0}", value), e);
RemoveTask(iterator.CurrentKey, task.Index, type);
continue;
}
updateMaxTaskId(currentId);
totalKeysToProcess += existingTask.NumberOfKeys;
task.Merge(existingTask);
RemoveTask(iterator.CurrentKey, task.Index, type);
} while (iterator.MoveNext());
}
}
private void MergeSimilarTasks(DatabaseTask task, byte[] taskId)
{
var id = Etag.Parse(taskId);
var type = task.GetType().FullName;
var tasksByIndexAndType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByIndexAndType);
using (var iterator = tasksByIndexAndType.MultiRead(Snapshot, (Slice)CreateKey(task.Index, type)))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
return;
}
int totalTaskCount = 0;
do
{
var currentId = Etag.Parse(iterator.CurrentKey.ToString());
if (currentId == id)
{
continue;
}
ushort version;
var value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null)
{
continue;
}
DatabaseTask existingTask;
try
{
existingTask = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
}
catch (Exception e)
{
Logger.ErrorException(
string.Format("Could not create instance of a task: {0}", value),
e);
RemoveTask(iterator.CurrentKey, task.Index, type);
continue;
}
task.Merge(existingTask);
RemoveTask(iterator.CurrentKey, task.Index, type);
if (totalTaskCount++ > 1024)
{
break;
}
}while (iterator.MoveNext());
}
}
public T GetMergedTask <T>() where T : DatabaseTask
{
var type = CreateKey(typeof(T).FullName);
var tasksByType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByType);
using (var iterator = tasksByType.MultiRead(Snapshot, (Slice)type))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
return(null);
}
do
{
ushort version;
var value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null)
{
continue;
}
DatabaseTask task;
try
{
task = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
}
catch (Exception e)
{
Logger.ErrorException(
string.Format("Could not create instance of a task: {0}", value),
e);
continue;
}
MergeSimilarTasks(task, value.ReadBytes(TaskFields.TaskId));
RemoveTask(iterator.CurrentKey, task.Index, type);
return((T)task);
}while (iterator.MoveNext());
}
return(null);
}
public int DeleteTasksForIndex(int indexId)
{
var count = 0;
var tasksByIndexAndType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByIndex);
using (var iterator = tasksByIndexAndType.MultiRead(Snapshot, (Slice)CreateKey(indexId)))
{
if (iterator.Seek(Slice.BeforeAllKeys) == false)
{
return(count);
}
do
{
ushort version;
var value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null)
{
continue;
}
DatabaseTask task;
try
{
task = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
}
catch (Exception e)
{
Logger.ErrorException(
string.Format("Could not create instance of a task: {0}, for deletion", value),
e);
continue;
}
var type = task.GetType().FullName;
RemoveTask(iterator.CurrentKey, task.Index, type);
count++;
} while (iterator.MoveNext());
}
return(count);
}
public T GetMergedTask <T>() where T : DatabaseTask
{
Api.MoveBeforeFirst(session, Tasks);
while (Api.TryMoveNext(session, Tasks))
{
var taskType = Api.RetrieveColumnAsString(session, Tasks, tableColumnsCache.TasksColumns["task_type"], Encoding.Unicode);
if (taskType != typeof(T).FullName)
{
continue;
}
var taskAsBytes = Api.RetrieveColumn(session, Tasks, tableColumnsCache.TasksColumns["task"]);
try
{
Api.JetDelete(session, Tasks);
}
catch (EsentErrorException e)
{
if (e.Error != JET_err.WriteConflict)
{
throw;
}
}
DatabaseTask task;
try
{
task = DatabaseTask.ToTask(taskType, taskAsBytes);
}
catch (Exception e)
{
logger.ErrorException(
string.Format("Could not create instance of a task: {0}", taskAsBytes),
e);
continue;
}
MergeSimilarTasks(task);
return((T)task);
}
return(null);
}
public T GetMergedTask <T>(Func <IComparable, MaxTaskIdStatus> maxIdStatus,
Action <IComparable> updateMaxTaskId, Reference <bool> foundWork, List <int> idsToSkip)
where T : DatabaseTask
{
var type = CreateKey(typeof(T).FullName);
var tasksByType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByType);
using (var iterator = tasksByType.MultiRead(Snapshot, (Slice)type))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
return(null);
}
do
{
ushort version;
var value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null)
{
continue;
}
DatabaseTask task;
try
{
task = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
}
catch (Exception e)
{
Logger.ErrorException(
string.Format("Could not create instance of a task: {0}", value),
e);
continue;
}
if (idsToSkip.Contains(task.Index))
{
continue;
}
var currentId = Etag.Parse(value.ReadBytes(TaskFields.TaskId));
switch (maxIdStatus(currentId))
{
case MaxTaskIdStatus.ReachedMaxTaskId:
// we found work and next run the merge option will be enabled
foundWork.Value = true;
return(null);
case MaxTaskIdStatus.Updated:
MergeSimilarTasks(task, currentId, updateMaxTaskId);
break;
case MaxTaskIdStatus.MergeDisabled:
default:
// returning only one task without merging
break;
}
RemoveTask(iterator.CurrentKey, task.Index, type);
return((T)task);
} while (iterator.MoveNext());
}
return(null);
}
public T GetMergedTask <T>(List <int> indexesToSkip, int[] allIndexes, HashSet <IComparable> alreadySeen)
where T : DatabaseTask
{
var expectedTaskType = typeof(T).FullName;
Api.JetSetCurrentIndex(session, Tasks, "by_task_type");
Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Tasks, SeekGrbit.SeekEQ) == false)
{
return(null);
}
Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.NewKey);
Api.JetSetIndexRange(session, Tasks, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit);
do
{
var taskType = Api.RetrieveColumnAsString(session, Tasks, tableColumnsCache.TasksColumns["task_type"], Encoding.ASCII);
// esent index ranges are approximate, and we need to check them ourselves as well
if (taskType != expectedTaskType)
{
//this shouldn't happen
logger.Warn("Tasks type mismatch: expected task type: {0}, current task type: {1}",
expectedTaskType, taskType);
continue;
}
var currentId = Api.RetrieveColumnAsInt32(session, Tasks, tableColumnsCache.TasksColumns["id"]).Value;
var index = Api.RetrieveColumnAsInt32(session, Tasks, tableColumnsCache.TasksColumns["for_index"]).Value;
if (indexesToSkip.Contains(index))
{
if (logger.IsDebugEnabled)
{
logger.Debug("Skipping task id: {0} for index id: {1}", currentId, index);
}
continue;
}
if (alreadySeen.Add(currentId) == false)
{
continue;
}
if (allIndexes.Contains(index) == false)
{
if (logger.IsDebugEnabled)
{
logger.Debug("Skipping task id: {0} for non existing index id: {0}", currentId, index);
}
continue;
}
var taskAsBytes = Api.RetrieveColumn(session, Tasks, tableColumnsCache.TasksColumns["task"]);
DatabaseTask task;
try
{
task = DatabaseTask.ToTask(taskType, taskAsBytes);
}
catch (Exception e)
{
logger.ErrorException(
string.Format("Could not create instance of a task: {0}", taskAsBytes),
e);
alreadySeen.Add(currentId);
continue;
}
if (logger.IsDebugEnabled)
{
logger.Debug("Fetched task id: {0}", currentId);
}
task.Id = currentId;
MergeSimilarTasks(task, alreadySeen, indexesToSkip, allIndexes);
return((T)task);
} while (Api.TryMoveNext(session, Tasks));
return(null);
}
private void MergeSimilarTasks(DatabaseTask task, HashSet <IComparable> alreadySeen, List <int> indexesToSkip, int[] allIndexes)
{
var expectedTaskType = task.GetType().FullName;
if (task.SeparateTasksByIndex)
{
Api.JetSetCurrentIndex(session, Tasks, "by_index_and_task_type");
Api.MakeKey(session, Tasks, task.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.None);
if (Api.TrySeek(session, Tasks, SeekGrbit.SeekEQ) == false)
{
// there are no tasks matching the current one, just return
return;
}
Api.MakeKey(session, Tasks, task.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.None);
Api.JetSetIndexRange(session, Tasks, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit);
}
else
{
Api.JetSetCurrentIndex(session, Tasks, "by_task_type");
Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Tasks, SeekGrbit.SeekEQ) == false)
{
// there are no tasks matching the current one, just return
return;
}
Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.NewKey);
Api.JetSetIndexRange(session, Tasks, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit);
}
var totalKeysToProcess = task.NumberOfKeys;
do
{
if (totalKeysToProcess >= 5 * 1024)
{
break;
}
var taskType = Api.RetrieveColumnAsString(session, Tasks, tableColumnsCache.TasksColumns["task_type"], Encoding.ASCII);
// esent index ranges are approximate, and we need to check them ourselves as well
if (taskType != expectedTaskType)
{
//this shouldn't happen
logger.Warn("Tasks type mismatch: expected task type: {0}, current task type: {1}",
expectedTaskType, taskType);
continue;
}
var currentId = Api.RetrieveColumnAsInt32(session, Tasks, tableColumnsCache.TasksColumns["id"]).Value;
var index = Api.RetrieveColumnAsInt32(session, Tasks, tableColumnsCache.TasksColumns["for_index"]).Value;
if (task.SeparateTasksByIndex == false && indexesToSkip.Contains(index))
{
//need to check this only when not separating tasks by index
if (logger.IsDebugEnabled)
{
logger.Debug("Skipping task id: {0} for index id: {1}", currentId, index);
}
continue;
}
if (alreadySeen.Add(currentId) == false)
{
continue;
}
if (task.SeparateTasksByIndex == false && allIndexes.Contains(index) == false)
{
//need to check this only when not separating tasks by index
if (logger.IsDebugEnabled)
{
logger.Debug("Skipping task id: {0} for non existing index id: {0}", currentId, index);
}
continue;
}
var taskAsBytes = Api.RetrieveColumn(session, Tasks, tableColumnsCache.TasksColumns["task"]);
DatabaseTask existingTask;
try
{
existingTask = DatabaseTask.ToTask(taskType, taskAsBytes);
}
catch (Exception e)
{
logger.ErrorException(
string.Format("Could not create instance of a task: {0}", taskAsBytes),
e);
alreadySeen.Add(currentId);
continue;
}
totalKeysToProcess += existingTask.NumberOfKeys;
task.Merge(existingTask);
if (logger.IsDebugEnabled)
{
logger.Debug("Merged task id: {0} with task id: {1}", currentId, task.Id);
}
} while (Api.TryMoveNext(session, Tasks));
}
public void MergeSimilarTasks(DatabaseTask task)
{
var expectedTaskType = task.GetType().FullName;
Api.JetSetCurrentIndex(session, Tasks, "by_index_and_task_type");
if (task.SeparateTasksByIndex)
{
Api.MakeKey(session, Tasks, task.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, Tasks, expectedTaskType, Encoding.Unicode, MakeKeyGrbit.None);
// there are no tasks matching the current one, just return
if (Api.TrySeek(session, Tasks, SeekGrbit.SeekEQ) == false)
{
return;
}
Api.MakeKey(session, Tasks, task.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, Tasks, expectedTaskType, Encoding.Unicode, MakeKeyGrbit.None);
Api.JetSetIndexRange(session, Tasks, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit);
}
else
{
if (Api.TryMoveFirst(session, Tasks) == false)
{
return;
}
}
int totalTaskCount = 0;
do
{
// esent index ranges are approximate, and we need to check them ourselves as well
if (Api.RetrieveColumnAsString(session, Tasks, tableColumnsCache.TasksColumns["task_type"]) != expectedTaskType)
{
continue;
}
try
{
var taskAsBytes = Api.RetrieveColumn(session, Tasks, tableColumnsCache.TasksColumns["task"]);
var taskType = Api.RetrieveColumnAsString(session, Tasks, tableColumnsCache.TasksColumns["task_type"], Encoding.Unicode);
DatabaseTask existingTask;
try
{
existingTask = DatabaseTask.ToTask(taskType, taskAsBytes);
}
catch (Exception e)
{
logger.ErrorException(
string.Format("Could not create instance of a task: {0}", taskAsBytes),
e);
Api.JetDelete(session, Tasks);
continue;
}
task.Merge(existingTask);
Api.JetDelete(session, Tasks);
}
catch (EsentErrorException e)
{
if (e.Error == JET_err.WriteConflict)
{
continue;
}
throw;
}
totalTaskCount++;
} while (Api.TryMoveNext(session, Tasks) && totalTaskCount < 1024);
}
public T GetMergedTask <T>(Func <IComparable, MaxTaskIdStatus> maxIdStatus,
Action <IComparable> updateMaxTaskId, Reference <bool> foundWork)
where T : DatabaseTask
{
Api.MoveBeforeFirst(session, Tasks);
while (Api.TryMoveNext(session, Tasks))
{
var taskType = Api.RetrieveColumnAsString(session, Tasks, tableColumnsCache.TasksColumns["task_type"], Encoding.Unicode);
if (taskType != typeof(T).FullName)
{
continue;
}
var taskAsBytes = Api.RetrieveColumn(session, Tasks, tableColumnsCache.TasksColumns["task"]);
DatabaseTask task;
try
{
task = DatabaseTask.ToTask(taskType, taskAsBytes);
}
catch (Exception e)
{
logger.ErrorException(
string.Format("Could not create instance of a task: {0}", taskAsBytes),
e);
continue;
}
var currentId = Api.RetrieveColumnAsInt32(session, Tasks, tableColumnsCache.TasksColumns["id"]).Value;
try
{
Api.JetDelete(session, Tasks);
}
catch (EsentErrorException e)
{
if (e.Error != JET_err.WriteConflict)
{
throw;
}
}
switch (maxIdStatus(currentId))
{
case MaxTaskIdStatus.ReachedMaxTaskId:
// we found work and next run the merge option will be enabled
foundWork.Value = true;
return(null);
case MaxTaskIdStatus.Updated:
MergeSimilarTasks(task, updateMaxTaskId);
break;
case MaxTaskIdStatus.MergeDisabled:
default:
// returning only one task without merging
break;
}
return((T)task);
}
return(null);
}
public T GetMergedTask <T>(List <int> indexesToSkip, int[] allIndexes, HashSet <IComparable> alreadySeen)
where T : DatabaseTask
{
var type = CreateKey(typeof(T).FullName);
var tasksByType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByType);
using (var iterator = tasksByType.MultiRead(Snapshot, (Slice)type))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
return(null);
}
do
{
ushort version;
var value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null)
{
continue;
}
var currentId = Etag.Parse(value.ReadBytes(TaskFields.TaskId));
var indexId = value.ReadInt(TaskFields.IndexId);
if (indexesToSkip.Contains(indexId))
{
if (Logger.IsDebugEnabled)
{
Logger.Debug("Skipping task id: {0} for index id: {1}", currentId, indexId);
}
continue;
}
if (alreadySeen.Add(currentId) == false)
{
continue;
}
if (allIndexes.Contains(indexId) == false)
{
if (Logger.IsDebugEnabled)
{
Logger.Debug("Skipping task id: {0} for non existing index id: {0}", currentId, indexId);
}
continue;
}
DatabaseTask task;
try
{
task = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
}
catch (Exception e)
{
Logger.ErrorException(
string.Format("Could not create instance of a task: {0}", value), e);
alreadySeen.Add(currentId);
continue;
}
if (Logger.IsDebugEnabled)
{
Logger.Debug("Fetched task id: {0}", currentId);
}
task.Id = currentId;
MergeSimilarTasks(task, alreadySeen, indexesToSkip, allIndexes);
return((T)task);
} while (iterator.MoveNext());
}
return(null);
}
private void MergeSimilarTasks(DatabaseTask task, HashSet <IComparable> alreadySeen, List <int> indexesToSkip, int[] allIndexes)
{
string tree;
Slice slice;
var type = task.GetType().FullName;
if (task.SeparateTasksByIndex)
{
tree = Tables.Tasks.Indices.ByIndexAndType;
slice = (Slice)CreateKey(task.Index, type);
}
else
{
tree = Tables.Tasks.Indices.ByType;
slice = (Slice)CreateKey(type);
}
using (var iterator = tableStorage.Tasks.GetIndex(tree).MultiRead(Snapshot, slice))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
return;
}
var totalKeysToProcess = task.NumberOfKeys;
do
{
if (totalKeysToProcess >= 5 * 1024)
{
break;
}
ushort version;
var value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null)
{
continue;
}
var currentId = Etag.Parse(iterator.CurrentKey.ToString());
var indexId = value.ReadInt(TaskFields.IndexId);
if (task.SeparateTasksByIndex == false && indexesToSkip.Contains(indexId))
{
//need to check this only when not separating tasks by index
if (Logger.IsDebugEnabled)
{
Logger.Debug("Skipping task id: {0} for index id: {1}", currentId, indexId);
}
continue;
}
if (alreadySeen.Add(currentId) == false)
{
continue;
}
if (task.SeparateTasksByIndex == false && allIndexes.Contains(indexId) == false)
{
//need to check this only when not separating tasks by index
if (Logger.IsDebugEnabled)
{
Logger.Debug("Skipping task id: {0} for non existing index id: {0}", currentId, indexId);
}
continue;
}
DatabaseTask existingTask;
try
{
existingTask = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
}
catch (Exception e)
{
Logger.ErrorException(string.Format("Could not create instance of a task: {0}", value), e);
alreadySeen.Add(currentId);
continue;
}
totalKeysToProcess += existingTask.NumberOfKeys;
task.Merge(existingTask);
if (Logger.IsDebugEnabled)
{
Logger.Debug("Merged task id: {0} with task id: {1}", currentId, task.Id);
}
} while (iterator.MoveNext());
}
}