Example #1
0
        public void AddTask(DatabaseTask task, DateTime addedAt)
        {
            var tasksByType         = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByType);
            var tasksByIndex        = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByIndex);
            var tasksByIndexAndType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByIndexAndType);

            var type       = task.GetType().FullName;
            var index      = task.Index;
            var id         = generator.CreateSequentialUuid(UuidType.Tasks);
            var idAsString = id.ToString();

            tableStorage.Tasks.Add(
                writeBatch.Value,
                idAsString,
                new RavenJObject
            {
                { "index", index },
                { "id", id.ToByteArray() },
                { "time", addedAt },
                { "type", type },
                { "task", task.AsBytes() }
            }, 0);

            tasksByType.MultiAdd(writeBatch.Value, CreateKey(type), idAsString);
            tasksByIndex.MultiAdd(writeBatch.Value, CreateKey(index), idAsString);
            tasksByIndexAndType.MultiAdd(writeBatch.Value, CreateKey(index, type), idAsString);
        }
Example #2
0
        public void AddTask(DatabaseTask task, DateTime addedAt)
        {
            var tasksByType         = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByType);
            var tasksByIndex        = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByIndex);
            var tasksByIndexAndType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByIndexAndType);

            var type       = task.GetType().FullName;
            var index      = task.Index;
            var id         = generator.CreateSequentialUuid(UuidType.Tasks);
            var idAsString = (Slice)id.ToString();

            var taskStructure = new Structure <TaskFields>(tableStorage.Tasks.Schema)
                                .Set(TaskFields.IndexId, index)
                                .Set(TaskFields.TaskId, id.ToByteArray())
                                .Set(TaskFields.AddedAt, addedAt.ToBinary())
                                .Set(TaskFields.Type, type)
                                .Set(TaskFields.SerializedTask, task.AsBytes());

            tableStorage.Tasks.AddStruct(writeBatch.Value, idAsString, taskStructure, 0);

            var indexKey = CreateKey(index);

            tasksByType.MultiAdd(writeBatch.Value, (Slice)CreateKey(type), idAsString);
            tasksByIndex.MultiAdd(writeBatch.Value, (Slice)indexKey, idAsString);
            tasksByIndexAndType.MultiAdd(writeBatch.Value, (Slice)AppendToKey(indexKey, type), idAsString);
        }
Example #3
0
        private void MergeSimilarTasks(DatabaseTask task, Etag taskId, Action <IComparable> updateMaxTaskId)
        {
            var type = task.GetType().FullName;
            var tasksByIndexAndType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByIndexAndType);

            using (var iterator = tasksByIndexAndType.MultiRead(Snapshot, (Slice)CreateKey(task.Index, type)))
            {
                if (!iterator.Seek(Slice.BeforeAllKeys))
                {
                    return;
                }

                var totalKeysToProcess = task.NumberOfKeys;
                do
                {
                    if (totalKeysToProcess >= 5 * 1024)
                    {
                        break;
                    }

                    var currentId = Etag.Parse(iterator.CurrentKey.ToString());
                    // this is the same task that we are trying to merge
                    if (currentId == taskId)
                    {
                        continue;
                    }

                    ushort version;
                    var    value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
                    if (value == null)
                    {
                        continue;
                    }

                    DatabaseTask existingTask;
                    try
                    {
                        existingTask = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
                    }
                    catch (Exception e)
                    {
                        Logger.ErrorException(string.Format("Could not create instance of a task: {0}", value), e);

                        RemoveTask(iterator.CurrentKey, task.Index, type);
                        continue;
                    }

                    updateMaxTaskId(currentId);

                    totalKeysToProcess += existingTask.NumberOfKeys;
                    task.Merge(existingTask);
                    RemoveTask(iterator.CurrentKey, task.Index, type);
                } while (iterator.MoveNext());
            }
        }
Example #4
0
        private void MergeSimilarTasks(DatabaseTask task, byte[] taskId)
        {
            var id   = Etag.Parse(taskId);
            var type = task.GetType().FullName;
            var tasksByIndexAndType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByIndexAndType);

            using (var iterator = tasksByIndexAndType.MultiRead(Snapshot, (Slice)CreateKey(task.Index, type)))
            {
                if (!iterator.Seek(Slice.BeforeAllKeys))
                {
                    return;
                }

                int totalTaskCount = 0;

                do
                {
                    var currentId = Etag.Parse(iterator.CurrentKey.ToString());
                    if (currentId == id)
                    {
                        continue;
                    }

                    ushort version;
                    var    value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
                    if (value == null)
                    {
                        continue;
                    }
                    DatabaseTask existingTask;
                    try
                    {
                        existingTask = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
                    }
                    catch (Exception e)
                    {
                        Logger.ErrorException(
                            string.Format("Could not create instance of a task: {0}", value),
                            e);

                        RemoveTask(iterator.CurrentKey, task.Index, type);
                        continue;
                    }

                    task.Merge(existingTask);
                    RemoveTask(iterator.CurrentKey, task.Index, type);

                    if (totalTaskCount++ > 1024)
                    {
                        break;
                    }
                }while (iterator.MoveNext());
            }
        }
Example #5
0
        public void AddTask(DatabaseTask task, DateTime addedAt)
        {
            int actualBookmarkSize;
            var bookmark = new byte[bookmarkMost];

            using (var update = new Update(session, Tasks, JET_prep.Insert))
            {
                Api.SetColumn(session, Tasks, tableColumnsCache.TasksColumns["task"], task.AsBytes());
                Api.SetColumn(session, Tasks, tableColumnsCache.TasksColumns["for_index"], task.Index);
                Api.SetColumn(session, Tasks, tableColumnsCache.TasksColumns["task_type"], task.GetType().FullName, Encoding.ASCII);
                Api.SetColumn(session, Tasks, tableColumnsCache.TasksColumns["added_at"], addedAt.ToBinary());

                update.Save(bookmark, bookmark.Length, out actualBookmarkSize);
            }
            Api.JetGotoBookmark(session, Tasks, bookmark, actualBookmarkSize);
        }
Example #6
0
        private void MergeSimilarTasks(DatabaseTask task, HashSet <IComparable> alreadySeen, List <int> indexesToSkip, int[] allIndexes)
        {
            var expectedTaskType = task.GetType().FullName;

            if (task.SeparateTasksByIndex)
            {
                Api.JetSetCurrentIndex(session, Tasks, "by_index_and_task_type");

                Api.MakeKey(session, Tasks, task.Index, MakeKeyGrbit.NewKey);
                Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.None);
                if (Api.TrySeek(session, Tasks, SeekGrbit.SeekEQ) == false)
                {
                    // there are no tasks matching the current one, just return
                    return;
                }

                Api.MakeKey(session, Tasks, task.Index, MakeKeyGrbit.NewKey);
                Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.None);
                Api.JetSetIndexRange(session, Tasks, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit);
            }
            else
            {
                Api.JetSetCurrentIndex(session, Tasks, "by_task_type");

                Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.NewKey);
                if (Api.TrySeek(session, Tasks, SeekGrbit.SeekEQ) == false)
                {
                    // there are no tasks matching the current one, just return
                    return;
                }

                Api.MakeKey(session, Tasks, expectedTaskType, Encoding.ASCII, MakeKeyGrbit.NewKey);
                Api.JetSetIndexRange(session, Tasks, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit);
            }

            var totalKeysToProcess = task.NumberOfKeys;

            do
            {
                if (totalKeysToProcess >= 5 * 1024)
                {
                    break;
                }

                var taskType = Api.RetrieveColumnAsString(session, Tasks, tableColumnsCache.TasksColumns["task_type"], Encoding.ASCII);
                // esent index ranges are approximate, and we need to check them ourselves as well
                if (taskType != expectedTaskType)
                {
                    //this shouldn't happen
                    logger.Warn("Tasks type mismatch: expected task type: {0}, current task type: {1}",
                                expectedTaskType, taskType);
                    continue;
                }


                var currentId = Api.RetrieveColumnAsInt32(session, Tasks, tableColumnsCache.TasksColumns["id"]).Value;
                var index     = Api.RetrieveColumnAsInt32(session, Tasks, tableColumnsCache.TasksColumns["for_index"]).Value;
                if (task.SeparateTasksByIndex == false && indexesToSkip.Contains(index))
                {
                    //need to check this only when not separating tasks by index
                    if (logger.IsDebugEnabled)
                    {
                        logger.Debug("Skipping task id: {0} for index id: {1}", currentId, index);
                    }
                    continue;
                }

                if (alreadySeen.Add(currentId) == false)
                {
                    continue;
                }

                if (task.SeparateTasksByIndex == false && allIndexes.Contains(index) == false)
                {
                    //need to check this only when not separating tasks by index
                    if (logger.IsDebugEnabled)
                    {
                        logger.Debug("Skipping task id: {0} for non existing index id: {0}", currentId, index);
                    }

                    continue;
                }

                var          taskAsBytes = Api.RetrieveColumn(session, Tasks, tableColumnsCache.TasksColumns["task"]);
                DatabaseTask existingTask;
                try
                {
                    existingTask = DatabaseTask.ToTask(taskType, taskAsBytes);
                }
                catch (Exception e)
                {
                    logger.ErrorException(
                        string.Format("Could not create instance of a task: {0}", taskAsBytes),
                        e);

                    alreadySeen.Add(currentId);
                    continue;
                }

                totalKeysToProcess += existingTask.NumberOfKeys;
                task.Merge(existingTask);
                if (logger.IsDebugEnabled)
                {
                    logger.Debug("Merged task id: {0} with task id: {1}", currentId, task.Id);
                }
            } while (Api.TryMoveNext(session, Tasks));
        }
Example #7
0
        public void MergeSimilarTasks(DatabaseTask task)
        {
            var expectedTaskType = task.GetType().FullName;

            Api.JetSetCurrentIndex(session, Tasks, "by_index_and_task_type");


            if (task.SeparateTasksByIndex)
            {
                Api.MakeKey(session, Tasks, task.Index, MakeKeyGrbit.NewKey);
                Api.MakeKey(session, Tasks, expectedTaskType, Encoding.Unicode, MakeKeyGrbit.None);
                // there are no tasks matching the current one, just return
                if (Api.TrySeek(session, Tasks, SeekGrbit.SeekEQ) == false)
                {
                    return;
                }
                Api.MakeKey(session, Tasks, task.Index, MakeKeyGrbit.NewKey);
                Api.MakeKey(session, Tasks, expectedTaskType, Encoding.Unicode, MakeKeyGrbit.None);
                Api.JetSetIndexRange(session, Tasks, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit);
            }

            else
            {
                if (Api.TryMoveFirst(session, Tasks) == false)
                {
                    return;
                }
            }

            int totalTaskCount = 0;

            do
            {
                // esent index ranges are approximate, and we need to check them ourselves as well
                if (Api.RetrieveColumnAsString(session, Tasks, tableColumnsCache.TasksColumns["task_type"]) != expectedTaskType)
                {
                    continue;
                }

                try
                {
                    var          taskAsBytes = Api.RetrieveColumn(session, Tasks, tableColumnsCache.TasksColumns["task"]);
                    var          taskType    = Api.RetrieveColumnAsString(session, Tasks, tableColumnsCache.TasksColumns["task_type"], Encoding.Unicode);
                    DatabaseTask existingTask;
                    try
                    {
                        existingTask = DatabaseTask.ToTask(taskType, taskAsBytes);
                    }
                    catch (Exception e)
                    {
                        logger.ErrorException(
                            string.Format("Could not create instance of a task: {0}", taskAsBytes),
                            e);
                        Api.JetDelete(session, Tasks);
                        continue;
                    }
                    task.Merge(existingTask);
                    Api.JetDelete(session, Tasks);
                }
                catch (EsentErrorException e)
                {
                    if (e.Error == JET_err.WriteConflict)
                    {
                        continue;
                    }
                    throw;
                }
                totalTaskCount++;
            } while (Api.TryMoveNext(session, Tasks) && totalTaskCount < 1024);
        }
Example #8
0
        private void MergeSimilarTasks(DatabaseTask task, HashSet <IComparable> alreadySeen, List <int> indexesToSkip, int[] allIndexes)
        {
            string tree;
            Slice  slice;
            var    type = task.GetType().FullName;

            if (task.SeparateTasksByIndex)
            {
                tree  = Tables.Tasks.Indices.ByIndexAndType;
                slice = (Slice)CreateKey(task.Index, type);
            }
            else
            {
                tree  = Tables.Tasks.Indices.ByType;
                slice = (Slice)CreateKey(type);
            }

            using (var iterator = tableStorage.Tasks.GetIndex(tree).MultiRead(Snapshot, slice))
            {
                if (!iterator.Seek(Slice.BeforeAllKeys))
                {
                    return;
                }

                var totalKeysToProcess = task.NumberOfKeys;
                do
                {
                    if (totalKeysToProcess >= 5 * 1024)
                    {
                        break;
                    }

                    ushort version;
                    var    value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
                    if (value == null)
                    {
                        continue;
                    }

                    var currentId = Etag.Parse(iterator.CurrentKey.ToString());
                    var indexId   = value.ReadInt(TaskFields.IndexId);
                    if (task.SeparateTasksByIndex == false && indexesToSkip.Contains(indexId))
                    {
                        //need to check this only when not separating tasks by index
                        if (Logger.IsDebugEnabled)
                        {
                            Logger.Debug("Skipping task id: {0} for index id: {1}", currentId, indexId);
                        }
                        continue;
                    }

                    if (alreadySeen.Add(currentId) == false)
                    {
                        continue;
                    }

                    if (task.SeparateTasksByIndex == false && allIndexes.Contains(indexId) == false)
                    {
                        //need to check this only when not separating tasks by index
                        if (Logger.IsDebugEnabled)
                        {
                            Logger.Debug("Skipping task id: {0} for non existing index id: {0}", currentId, indexId);
                        }

                        continue;
                    }

                    DatabaseTask existingTask;
                    try
                    {
                        existingTask = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
                    }
                    catch (Exception e)
                    {
                        Logger.ErrorException(string.Format("Could not create instance of a task: {0}", value), e);
                        alreadySeen.Add(currentId);
                        continue;
                    }

                    totalKeysToProcess += existingTask.NumberOfKeys;
                    task.Merge(existingTask);
                    if (Logger.IsDebugEnabled)
                    {
                        Logger.Debug("Merged task id: {0} with task id: {1}", currentId, task.Id);
                    }
                } while (iterator.MoveNext());
            }
        }