private ObjectBlockRequest <T> ConvertToObjectBlockRequest <T>(IObjectBlockSettings <T> settings) { var request = new ObjectBlockRequest <T>(settings.Object, _taskConfiguration.MaxLengthForNonCompressedData); request.ApplicationName = _taskExecutionInstance.ApplicationName; request.TaskName = _taskExecutionInstance.TaskName; request.TaskExecutionId = _taskExecutionInstance.TaskExecutionId; request.TaskDeathMode = _taskExecutionOptions.TaskDeathMode; if (_taskExecutionOptions.TaskDeathMode == TaskDeathMode.KeepAlive) { request.KeepAliveDeathThreshold = _taskExecutionOptions.KeepAliveDeathThreshold.Value; } else { request.OverrideDeathThreshold = _taskExecutionOptions.OverrideThreshold.Value; } request.ReprocessReferenceValue = settings.ReferenceValueToReprocess; request.ReprocessOption = settings.ReprocessOption; SetConfigurationOverridableSettings(request, settings); return(request); }
public IList <IObjectBlockContext <T> > GenerateObjectBlocks <T>(ObjectBlockRequest <T> blockRequest) { var blocks = new List <IObjectBlockContext <T> >(); if (!string.IsNullOrEmpty(blockRequest.ReprocessReferenceValue)) { blocks = LoadObjectBlocksOfTask(blockRequest); } else { var forceBlocks = GetForcedObjectBlocks(blockRequest); blocks.AddRange(forceBlocks); if (GetBlocksRemaining(blockRequest, blocks) > 0) { LoadFailedAndDeadObjectBlocks(blockRequest, blocks); } if (GetBlocksRemaining(blockRequest, blocks) > 0 && blockRequest.Object != null) { var newBlock = GenerateNewObjectBlock(blockRequest); blocks.Add(newBlock); } } if (!blocks.Any()) { LogEmptyBlockEvent(blockRequest.TaskExecutionId, blockRequest.ApplicationName, blockRequest.TaskName); } return(blocks.OrderBy(x => long.Parse(x.Block.ObjectBlockId)).ToList()); }
private List <IObjectBlockContext <T> > GetDeadObjectBlocks <T>(ObjectBlockRequest <T> blockRequest, int blockCountLimit) { var deadBlockRequest = CreateDeadBlocksRequest(blockRequest, blockCountLimit); var deadBlocks = _blockRepository.FindDeadObjectBlocks <T>(deadBlockRequest); return(CreateObjectBlockContexts(blockRequest, deadBlocks)); }
private async Task <ObjectBlock <T> > GenerateObjectBlockAsync <T>(ObjectBlockRequest <T> blockRequest) { var request = new ObjectBlockCreateRequest <T>(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName), blockRequest.TaskExecutionId, blockRequest.Object, blockRequest.CompressionThreshold); var objectBlock = (await _blockRepository.AddObjectBlockAsync(request).ConfigureAwait(false)).Block; await Task.Delay(10).ConfigureAwait(false); // guarantee that each block has a unique created date return(objectBlock); }
private List <IObjectBlockContext <T> > CreateObjectBlockContexts <T>(ObjectBlockRequest <T> blockRequest, IList <ObjectBlock <T> > objectBlocks) { var blocks = new List <IObjectBlockContext <T> >(); foreach (var objectBlock in objectBlocks) { var blockContext = CreateObjectBlockContext(blockRequest, objectBlock); blocks.Add(blockContext); } return(blocks); }
private ObjectBlock <T> GenerateObjectBlock <T>(ObjectBlockRequest <T> blockRequest) { var request = new ObjectBlockCreateRequest <T>(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName), blockRequest.TaskExecutionId, blockRequest.Object, blockRequest.CompressionThreshold); var objectBlock = _blockRepository.AddObjectBlock(request).Block; Thread.Sleep(5); // guarantee that each block has a unique created date return(objectBlock); }
private List <IObjectBlockContext <T> > GetFailedObjectBlocks <T>(ObjectBlockRequest <T> blockRequest, int blockCountLimit) { var failedBlockRequest = new FindFailedBlocksRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName), blockRequest.TaskExecutionId, blockRequest.BlockType, DateTime.UtcNow - blockRequest.FailedTaskDetectionRange, DateTime.UtcNow, blockCountLimit, blockRequest.FailedTaskRetryLimit ); var failedBlocks = _blockRepository.FindFailedObjectBlocks <T>(failedBlockRequest); return(CreateObjectBlockContexts <T>(blockRequest, failedBlocks)); }
private async Task LoadFailedAndDeadObjectBlocksAsync <T>(ObjectBlockRequest <T> blockRequest, List <IObjectBlockContext <T> > blocks) { int blocksRemaining = GetBlocksRemaining(blockRequest, blocks); if (blockRequest.ReprocessDeadTasks) { blocks.AddRange(await GetDeadObjectBlocksAsync(blockRequest, blocksRemaining).ConfigureAwait(false)); blocksRemaining = blockRequest.MaxBlocks - blocks.Count; } if (GetBlocksRemaining(blockRequest, blocks) > 0 && blockRequest.ReprocessFailedTasks) { blocks.AddRange(await GetFailedObjectBlocksAsync(blockRequest, blocksRemaining).ConfigureAwait(false)); } }
private void LoadFailedAndDeadObjectBlocks <T>(ObjectBlockRequest <T> blockRequest, List <IObjectBlockContext <T> > blocks) { int blocksRemaining = GetBlocksRemaining(blockRequest, blocks); if (blockRequest.ReprocessDeadTasks) { blocks.AddRange(GetDeadObjectBlocks(blockRequest, blocksRemaining)); blocksRemaining = blockRequest.MaxBlocks - blocks.Count; } if (GetBlocksRemaining(blockRequest, blocks) > 0 && blockRequest.ReprocessFailedTasks) { blocks.AddRange(GetFailedObjectBlocks(blockRequest, blocksRemaining)); } }
private List <IObjectBlockContext <T> > LoadObjectBlocksOfTask <T>(ObjectBlockRequest <T> blockRequest) { var failedBlockRequest = new FindBlocksOfTaskRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName), blockRequest.TaskExecutionId, blockRequest.BlockType, blockRequest.ReprocessReferenceValue, blockRequest.ReprocessOption); var blocksOfTask = _blockRepository.FindObjectBlocksOfTask <T>(failedBlockRequest); if (!blocksOfTask.Any()) { return(new List <IObjectBlockContext <T> >()); } return(CreateObjectBlockContexts(blockRequest, blocksOfTask)); }
private IObjectBlockContext <T> CreateObjectBlockContext <T>(ObjectBlockRequest <T> blockRequest, ObjectBlock <T> objectBlock, int forcedBlockQueueId = 0) { var attempt = objectBlock.Attempt + 1; var createRequest = new BlockExecutionCreateRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName), blockRequest.TaskExecutionId, blockRequest.BlockType, objectBlock.ObjectBlockId, attempt); var blockExecutionId = _blockRepository.AddObjectBlockExecution(createRequest); var blockContext = new ObjectBlockContext <T>(_objectBlockRepository, _taskExecutionRepository, blockRequest.ApplicationName, blockRequest.TaskName, blockRequest.TaskExecutionId, objectBlock, blockExecutionId, forcedBlockQueueId.ToString()); return(blockContext); }
private List <IObjectBlockContext <T> > GetForcedObjectBlocks <T>(ObjectBlockRequest <T> blockRequest) { var forcedBlockRequest = new QueuedForcedBlocksRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName), blockRequest.TaskExecutionId, blockRequest.BlockType); var queuedForcedBlocks = _blockRepository.GetQueuedForcedObjectBlocks <T>(forcedBlockRequest); var forcedBlocks = new List <IObjectBlockContext <T> >(); foreach (var queuedForcedBlock in queuedForcedBlocks) { forcedBlocks.Add(CreateObjectBlockContext(blockRequest, queuedForcedBlock.ObjectBlock, queuedForcedBlock.ForcedBlockQueueId)); } if (forcedBlocks.Any()) { DequeueForcedBlocks(blockRequest, forcedBlocks.Select(x => x.ForcedBlockQueueId).ToList()); } return(forcedBlocks); }
private async Task <List <IObjectBlockContext <T> > > LoadObjectBlocksOfTaskAsync <T>(ObjectBlockRequest <T> blockRequest) { var failedBlockRequest = new FindBlocksOfTaskRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName), blockRequest.TaskExecutionId, blockRequest.BlockType, blockRequest.ReprocessReferenceValue, blockRequest.ReprocessOption); var blocksOfTask = await _blockRepository.FindObjectBlocksOfTaskAsync <T>(failedBlockRequest).ConfigureAwait(false); if (!blocksOfTask.Any()) { return(new List <IObjectBlockContext <T> >()); } return(await CreateObjectBlockContextsAsync(blockRequest, blocksOfTask).ConfigureAwait(false)); }
private async Task <List <IObjectBlockContext <T> > > GetDeadObjectBlocksAsync <T>(ObjectBlockRequest <T> blockRequest, int blockCountLimit) { var deadBlockRequest = CreateDeadBlocksRequest(blockRequest, blockCountLimit); var deadBlocks = await _blockRepository.FindDeadObjectBlocksAsync <T>(deadBlockRequest).ConfigureAwait(false); return(await CreateObjectBlockContextsAsync(blockRequest, deadBlocks).ConfigureAwait(false)); }
private IObjectBlockContext <T> GenerateNewObjectBlock <T>(ObjectBlockRequest <T> blockRequest) { var newObjectBlock = GenerateObjectBlock(blockRequest); return(CreateObjectBlockContext(blockRequest, newObjectBlock)); }
private async Task <List <IObjectBlockContext <T> > > GetFailedObjectBlocksAsync <T>(ObjectBlockRequest <T> blockRequest, int blockCountLimit) { var failedBlockRequest = new FindFailedBlocksRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName), blockRequest.TaskExecutionId, blockRequest.BlockType, DateTime.UtcNow - blockRequest.FailedTaskDetectionRange, DateTime.UtcNow, blockCountLimit, blockRequest.FailedTaskRetryLimit ); var failedBlocks = await _blockRepository.FindFailedObjectBlocksAsync <T>(failedBlockRequest).ConfigureAwait(false); return(await CreateObjectBlockContextsAsync <T>(blockRequest, failedBlocks).ConfigureAwait(false)); }
private async Task <List <IObjectBlockContext <T> > > CreateObjectBlockContextsAsync <T>(ObjectBlockRequest <T> blockRequest, IList <ObjectBlock <T> > objectBlocks) { var blocks = new List <IObjectBlockContext <T> >(); foreach (var objectBlock in objectBlocks) { var blockContext = await CreateObjectBlockContextAsync(blockRequest, objectBlock).ConfigureAwait(false); blocks.Add(blockContext); } return(blocks); }
private async Task <IObjectBlockContext <T> > GenerateNewObjectBlockAsync <T>(ObjectBlockRequest <T> blockRequest) { var newObjectBlock = await GenerateObjectBlockAsync(blockRequest).ConfigureAwait(false); return(await CreateObjectBlockContextAsync(blockRequest, newObjectBlock).ConfigureAwait(false)); }
private async Task <List <IObjectBlockContext <T> > > GetForcedObjectBlocksAsync <T>(ObjectBlockRequest <T> blockRequest) { var forcedBlockRequest = new QueuedForcedBlocksRequest(new TaskId(blockRequest.ApplicationName, blockRequest.TaskName), blockRequest.TaskExecutionId, blockRequest.BlockType); var queuedForcedBlocks = await _blockRepository.GetQueuedForcedObjectBlocksAsync <T>(forcedBlockRequest).ConfigureAwait(false); var forcedBlocks = new List <IObjectBlockContext <T> >(); foreach (var queuedForcedBlock in queuedForcedBlocks) { forcedBlocks.Add(await CreateObjectBlockContextAsync(blockRequest, queuedForcedBlock.ObjectBlock, queuedForcedBlock.ForcedBlockQueueId).ConfigureAwait(false)); } if (forcedBlocks.Any()) { await DequeueForcedBlocksAsync(blockRequest, forcedBlocks.Select(x => x.ForcedBlockQueueId).ToList()).ConfigureAwait(false); } return(forcedBlocks); }